Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6fdcfad8d3 | ||
|
|
8d9d33ae85 | ||
|
|
cc25baa563 | ||
|
|
68a2ebdcc3 | ||
|
|
69b99eec68 | ||
|
|
09bd0eec4f | ||
|
|
222a38e050 | ||
|
|
ee99a24f77 | ||
|
|
f68521da28 | ||
|
|
296064c799 | ||
|
|
df9231c734 | ||
|
|
843b709b59 | ||
|
|
f506fb34ec |
@@ -676,6 +676,9 @@ type answers struct {
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
|
||||
// journal was the only place that said so for a day (04-ISSUES/179).
|
||||
failing []inventory.ProviderStanding
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||
//
|
||||
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||
// answers is a lookup that waits out its timeout on every name.
|
||||
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||
var replicated []catalogue.Seat
|
||||
for _, s := range catalogue.Seats() {
|
||||
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||
replicated = append(replicated, s)
|
||||
}
|
||||
}
|
||||
if len(replicated) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, seat := range replicated {
|
||||
var nodes []string
|
||||
for _, h := range recorded {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||
nodes = append(nodes, h.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
var derived []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
derived = append(derived, e.On...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(derived) == 1 {
|
||||
nodes = derived
|
||||
}
|
||||
}
|
||||
at := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if address[n] != "" {
|
||||
at[overlay.InternalName(n)] = address[n]
|
||||
}
|
||||
}
|
||||
out[seat.Name] = at
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -194,6 +194,7 @@ func usage() {
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
|
||||
@@ -661,7 +661,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
|
||||
@@ -505,6 +505,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
|
||||
// capabilities, because it is something not working now and they are a description.
|
||||
failing, err := failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if here := failingOn(failing, name); len(here) > 0 {
|
||||
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
|
||||
for _, line := range failingLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -707,6 +707,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
|
||||
// lists every holder of the mesh's resolver.
|
||||
replicas, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
@@ -783,7 +790,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines, Zones: zones,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||
|
||||
@@ -132,6 +132,11 @@ func serve(ctx context.Context) error {
|
||||
if err := server.Answers(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||
// 0224): a provider's journal must not be the only place that says so.
|
||||
if err := server.Watches(standings{inv}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
||||
// from the store's row, so what the seat declares is what is answered.
|
||||
|
||||
@@ -78,6 +78,11 @@ type meshStatus struct {
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
||||
// document without this called the mesh well while the identity provider refused every consumer
|
||||
// for a day (04-ISSUES/179).
|
||||
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -210,6 +215,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.Failing = asked.failing
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||
// would bury the ones that matter under a list nobody can act on.
|
||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||
}
|
||||
|
||||
|
||||
@@ -29,11 +29,13 @@ type seatHolder struct {
|
||||
|
||||
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||
type seatRow struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Holders: []seatHolder{}}
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
if len(args) == 3 && args[1] == "--to" {
|
||||
return handOver(ctx, args[0], args[2])
|
||||
return handOver(ctx, args[0], args[2], false)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
|
||||
if len(args) == 3 && args[1] == "--add" {
|
||||
return handOver(ctx, args[0], args[2], true)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
|
||||
"seat <name> --add <node>/<module>")
|
||||
}
|
||||
|
||||
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
|
||||
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||
// and refusing to record a handover to a module the node has not started would make the handover
|
||||
// impossible to do before the switch instead of as the switch.
|
||||
func handOver(ctx context.Context, seatName, to string) error {
|
||||
//
|
||||
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
|
||||
// records the named assignment as a further holder and leaves every holder on record as it is. A
|
||||
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
|
||||
// the one named, as it always did.
|
||||
func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||
nodeName, module, ok := strings.Cut(to, "/")
|
||||
if !ok || nodeName == "" || module == "" {
|
||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
if !known {
|
||||
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||
}
|
||||
if adding && !seat.Replicated {
|
||||
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
|
||||
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||
}
|
||||
var was string
|
||||
var held []string
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||
was = h.Node
|
||||
held = append(held, h.Node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||
}
|
||||
if adding {
|
||||
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
|
||||
strings.Join(held, ", "))
|
||||
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
|
||||
return nil
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
|
||||
//
|
||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
|
||||
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
|
||||
|
||||
// standings keeps what providers say, in the inventory.
|
||||
type standings struct{ inv *inventory.Inventory }
|
||||
|
||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
|
||||
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
|
||||
Consumer: st.Consumer, ConsumerNode: st.Node,
|
||||
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
|
||||
})
|
||||
}
|
||||
|
||||
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
|
||||
//
|
||||
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
|
||||
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
|
||||
// consumer clears it.
|
||||
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
|
||||
all, err := inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
|
||||
}
|
||||
assigned := map[string]map[string]bool{}
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range all {
|
||||
on, asked := assigned[s.ProviderNode]
|
||||
if !asked {
|
||||
modules, err := inv.Assigned(ctx, s.ProviderNode)
|
||||
if err != nil {
|
||||
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
|
||||
modules = nil
|
||||
}
|
||||
on = map[string]bool{}
|
||||
for _, m := range modules {
|
||||
on[m] = true
|
||||
}
|
||||
assigned[s.ProviderNode] = on
|
||||
}
|
||||
if on[s.Module] {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
|
||||
// that stopped repeating itself said so.
|
||||
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
|
||||
var out []string
|
||||
for _, s := range list {
|
||||
where := s.Module
|
||||
if s.ProviderNode != "" {
|
||||
where += " on " + s.ProviderNode
|
||||
}
|
||||
whom := s.Consumer
|
||||
if s.ConsumerNode != "" {
|
||||
whom += " (" + s.ConsumerNode + ")"
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
|
||||
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
|
||||
s.Since.Local().Format("2006-01-02 15:04")))
|
||||
if e := strings.TrimSpace(s.Error); e != "" {
|
||||
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
|
||||
}
|
||||
if s.Quiet(now) {
|
||||
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
|
||||
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orUnclassed(class string) string {
|
||||
if class == "" {
|
||||
return "failing"
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// printFailing is the status section, said when there is anything to say.
|
||||
func printFailing(list []inventory.ProviderStanding, now time.Time) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
|
||||
for _, line := range failingLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
|
||||
}
|
||||
|
||||
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
|
||||
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range list {
|
||||
if s.ProviderNode == node || s.ConsumerNode == node {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
|
||||
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
|
||||
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
|
||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
|
||||
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
|
||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept := standings{open.inventory}
|
||||
since := time.Now().Add(-23 * time.Hour)
|
||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
|
||||
if _, err := kept.Stood(ctx, failing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
|
||||
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
if strings.Contains(said, "all doing what they were told") {
|
||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Failing []inventory.ProviderStanding `json:"failing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovered: gone, and the mesh may be well again as far as this is concerned.
|
||||
failing.Failing = false
|
||||
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
|
||||
t.Fatalf("%v %v", cleared, err)
|
||||
}
|
||||
asked, err = theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||
// not a problem.
|
||||
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("%+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
|
||||
now := time.Now()
|
||||
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
|
||||
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
|
||||
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
|
||||
}}, now), "\n")
|
||||
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Fatalf("%q not in:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lines, "more") {
|
||||
t.Fatalf("more than the first line of an error:\n%s", lines)
|
||||
}
|
||||
}
|
||||
@@ -129,6 +129,10 @@ func printStatus(asked answers) error {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// A provider failing a consumer, beside machines failing what they were told: both are something
|
||||
// not working now (novox/hq ADR 0224).
|
||||
printFailing(asked.failing, time.Now())
|
||||
|
||||
if len(quiet) > 0 {
|
||||
var said []string
|
||||
for _, n := range quiet {
|
||||
@@ -416,6 +420,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
}
|
||||
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
|
||||
// providers announced: nothing else in the mesh knows whether a provision is being made.
|
||||
out.failing, err = failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
@@ -528,7 +538,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
|
||||
@@ -253,10 +253,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
|
||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
// controller a subscriber to every event in the mesh, and its permission list would stop
|
||||
// saying what it is for. The ack grant below is scoped per stream because the controller's
|
||||
// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
|
||||
// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
|
||||
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||
// ever, refused by the list it already has.
|
||||
|
||||
@@ -5,16 +5,16 @@ import "testing"
|
||||
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
|
||||
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
|
||||
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
|
||||
seat := Seat{Name: "node-hosts-file", Scope: "node", Serves: []string{"entries"}}
|
||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hosts", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hosts", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries.one")
|
||||
has(t, two.Subscribe, "mesh.seat.node-hosts-file.tool.entries.two")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hosts-file.tool.entries.two")
|
||||
seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
|
||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
|
||||
has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||
|
||||
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, user.Publish, "mesh.seat.node-hosts-file.tool.entries.*")
|
||||
has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
|
||||
}
|
||||
|
||||
// A mesh-scoped seat's tool stays flat: nothing about it changes.
|
||||
|
||||
@@ -226,8 +226,23 @@ var ControllerFollows = []string{
|
||||
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
||||
// with the retired seat row.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
|
||||
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
|
||||
// from whichever module provides — because the rule is about every provider, and a list of
|
||||
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
|
||||
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
|
||||
// the index is a name.
|
||||
moduleEventSubject("*", ProvisionerFailing),
|
||||
moduleEventSubject("*", ProvisionerRecovered),
|
||||
}
|
||||
|
||||
// The provider standing events, by their local names. Written here as well as in the catalogue
|
||||
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
||||
func moduleEventSubject(module, event string) string {
|
||||
@@ -271,7 +286,7 @@ func MeshConsumers() []Consumer {
|
||||
// client and come back to be acted on again.
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
|
||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
},
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ accounts {
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
|
||||
@@ -159,6 +159,11 @@ type Rendering struct {
|
||||
// 0199): the mesh's resolver forwards each one there.
|
||||
Zones []ZoneAt
|
||||
|
||||
// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
|
||||
// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
|
||||
// file lists: every holder of the mesh's resolver, this machine first if it is one.
|
||||
Holders map[string]map[string]string
|
||||
|
||||
Settings SettingsBy
|
||||
Generators map[string]Generator
|
||||
// Grants are the credentials this node must create, for the provisions it offers. Passed in
|
||||
@@ -980,7 +985,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||
// it declares.
|
||||
given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
|
||||
given, err := FactsFrom(m, r, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package catalogue
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -159,3 +160,34 @@ func consumePattern(pattern string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
|
||||
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
|
||||
// controller follows them from every module and `status` names a consumer failing until it recovers.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// ProvisionerEvents are both, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
|
||||
|
||||
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
|
||||
// contributions — a provider, running a provisioner over them — the provider's standing events.
|
||||
//
|
||||
// **Derived, not declared**, because they are the mesh's rule about every provider rather than
|
||||
// anything one module chose to say: a provider whose manifest forgot them would fail its consumers
|
||||
// as silently as on 2026-10-05, with its announcement refused by the bus (novox/hq issue 179). Every
|
||||
// grant of a module's publishing reads this, never the declared list alone.
|
||||
func (m Manifest) EmitsAll() []string {
|
||||
out := append([]string(nil), m.Emits...)
|
||||
if len(m.Receives) == 0 {
|
||||
return out
|
||||
}
|
||||
for _, e := range ProvisionerEvents {
|
||||
if !slices.Contains(out, e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0223 part 3: a machine's names are one seat's. The `hosts` module holding
|
||||
// `node-hosts-file` became `hostname` holding `node-hostname`, which writes /etc/hostname beside the
|
||||
// machine's own lines in /etc/hosts. The seat was renamed (ADR 0122), so what claims the old name — a
|
||||
// manifest registered before the rename — still holds the one seat.
|
||||
|
||||
func withHostnameAlias(t *testing.T) {
|
||||
t.Helper()
|
||||
was := aliases
|
||||
t.Cleanup(func() { aliases = was })
|
||||
UseAliases(map[string]string{"node-hosts-file": "node-hostname"})
|
||||
}
|
||||
|
||||
func TestTheHostsFilesFormerNameResolvesToTheHostnameSeat(t *testing.T) {
|
||||
if _, known := SeatNamed("node-hostname"); !known {
|
||||
t.Fatal("node-hostname is not in the mesh's set")
|
||||
}
|
||||
withHostnameAlias(t)
|
||||
seat, known := SeatNamed("node-hosts-file")
|
||||
if !known || seat.Name != "node-hostname" || seat.Scope != ScopeNode {
|
||||
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
|
||||
}
|
||||
var verbs []string
|
||||
for _, v := range seat.Serves {
|
||||
verbs = append(verbs, v.Name)
|
||||
}
|
||||
if strings.Join(verbs, " ") != "entries add remove" {
|
||||
t.Errorf("the renamed seat serves %v; its verbs are unchanged", verbs)
|
||||
}
|
||||
}
|
||||
|
||||
// One seat under either name: the module registered before the rename and the one after cannot both
|
||||
// hold it on one machine.
|
||||
func TestTheOldAndTheNewClaimantAreOneSeatOnAMachine(t *testing.T) {
|
||||
withHostnameAlias(t)
|
||||
cat := shelf(
|
||||
mod("hosts", nil, nil, nil, Claim{Name: "node-hosts-file"}),
|
||||
mod("hostname", nil, nil, nil, Claim{Name: "node-hostname"}),
|
||||
)
|
||||
_, err := Resolve(cat, []string{"hosts", "hostname"}, workstation(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "node-hostname") {
|
||||
t.Errorf("hosts and hostname both held the machine's names on one machine: %v", err)
|
||||
}
|
||||
if _, err := Resolve(cat, []string{"hosts"}, workstation(), World{}); err != nil {
|
||||
t.Errorf("a machine still assigned hosts under the old name does not resolve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's module: /etc/hostname is the operator's `hostname` setting. Without it the module is
|
||||
// left out, naming the key — the mesh never renames a machine on its own — and a mesh-wide setting
|
||||
// naming ${machine:name} gives every machine its mesh name.
|
||||
func TestTheMachinesNameIsItsSetting(t *testing.T) {
|
||||
cat := map[string]Manifest{"hostname": catalogueManifest(t, "hostname")}
|
||||
got, err := Resolve(cat, []string{"hostname"}, Node{Name: "ace", At: "ace.internal"}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
machines := map[string]string{"ace.internal": "10.42.0.2"}
|
||||
nameOf := func(settings SettingsBy) (string, string) {
|
||||
t.Helper()
|
||||
composed, err := got.Compose(Rendering{Names: machines, Machines: machines, Suffix: "internal",
|
||||
Settings: settings})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := composed.LeftOut["hostname"]; why != "" {
|
||||
return "", why
|
||||
}
|
||||
for _, r := range composed.Resources {
|
||||
if r["path"] == "/etc/hostname" {
|
||||
return r["content"].(string), ""
|
||||
}
|
||||
}
|
||||
t.Fatal("no /etc/hostname composed")
|
||||
return "", ""
|
||||
}
|
||||
|
||||
if _, why := nameOf(nil); !strings.Contains(why, "hostname") {
|
||||
t.Errorf("with no setting the machine's name was written, or left out for another reason: %q", why)
|
||||
}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {{From: "ace", Values: map[string]any{"hostname": "Ace"}}}}); name != "Ace\n" {
|
||||
t.Errorf("the operator's name for the machine gave %q (%s)", name, why)
|
||||
}
|
||||
mesh := Layer{From: "the mesh", Values: map[string]any{"hostname": "${machine:name}"}}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {mesh}}); name != "ace\n" {
|
||||
t.Errorf("a mesh-wide ${machine:name} gave %q (%s)", name, why)
|
||||
}
|
||||
node := Layer{From: "ace", Values: map[string]any{"hostname": "Ace"}}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {mesh, node}}); name != "Ace\n" {
|
||||
t.Errorf("a machine's own name over the mesh-wide one gave %q (%s)", name, why)
|
||||
}
|
||||
}
|
||||
@@ -767,16 +767,27 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
var problems []string
|
||||
var held []Held
|
||||
|
||||
// onRecord is the recorded holder of a seat, if a handover ever named one.
|
||||
onRecord := func(claim, scope string) (Held, bool) {
|
||||
// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
|
||||
// and as many as were added for a replicated one (novox/hq ADR 0223).
|
||||
onRecord := func(claim, scope string) []Held {
|
||||
var out []Held
|
||||
for _, h := range holdings {
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
cs, cok := SeatNamed(claim)
|
||||
if ok && cok && hs.Name == cs.Name && h.Scope == scope {
|
||||
return h, true
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return Held{}, false
|
||||
return out
|
||||
}
|
||||
// recordedHere says this node's module is one of a seat's holders on record.
|
||||
recordedHere := func(claim, scope, module string) bool {
|
||||
for _, rec := range onRecord(claim, scope) {
|
||||
if rec.Node == node.Name && rec.Module == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
byScope := map[string]map[string]string{} // scope → claim → module
|
||||
@@ -787,21 +798,35 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// the seat but is not the one on record is eligible, and that is all: it is not a second
|
||||
// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
|
||||
// lets the next holder stand beside the current one until the seat is handed over.
|
||||
if rec, recorded := onRecord(c.Name, scope); recorded {
|
||||
if rec.Node != node.Name || rec.Module != m.Module {
|
||||
if recs := onRecord(c.Name, scope); len(recs) > 0 {
|
||||
// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
|
||||
// may have several holders on record; several for any other seat is a store that
|
||||
// disagrees with the mesh's definition of the role, and it is refused, named, rather
|
||||
// than letting two machines answer for what the mesh has one of.
|
||||
if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%q is on record as held by %d assignments, and it is held once per %s — "+
|
||||
"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
|
||||
continue
|
||||
}
|
||||
if !recordedHere(c.Name, scope, m.Module) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if byScope[scope] == nil {
|
||||
byScope[scope] = map[string]string{}
|
||||
}
|
||||
if other, taken := byScope[scope][c.Name]; taken {
|
||||
// **One seat under either of its names** (novox/hq ADR 0122): a manifest registered before
|
||||
// a rename claims the former name, and one written after it the current — two claimants of
|
||||
// one seat, compared by the seat they resolve to and not by how each spelled it.
|
||||
seat := canonicalSeat(c.Name)
|
||||
if other, taken := byScope[scope][seat]; taken {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both claim %q, and only one thing may hold it per %s",
|
||||
other, m.Module, c.Name, scope))
|
||||
other, m.Module, seat, scope))
|
||||
continue
|
||||
}
|
||||
byScope[scope][c.Name] = m.Module
|
||||
byScope[scope][seat] = m.Module
|
||||
held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
|
||||
Module: m.Module, Site: node.Site})
|
||||
}
|
||||
@@ -810,11 +835,17 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// And against the rest of the mesh, for the scopes that reach past this machine.
|
||||
for _, h := range held {
|
||||
for _, e := range elsewhere {
|
||||
if e.Node == node.Name || e.Claim != h.Claim || e.Scope != h.Scope {
|
||||
if e.Node == node.Name || canonicalSeat(e.Claim) != canonicalSeat(h.Claim) || e.Scope != h.Scope {
|
||||
continue
|
||||
}
|
||||
switch h.Scope {
|
||||
case ScopeMesh:
|
||||
// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
|
||||
// the mesh's settled answer: one for most seats, several only for a replicated seat,
|
||||
// each added by an act. Two holders here are two records, and both hold.
|
||||
if recordedHere(h.Claim, h.Scope, h.Module) {
|
||||
continue
|
||||
}
|
||||
// Both claim and nobody is on record, or this refusal could not have happened.
|
||||
// The remedy is the handover that records the holder (novox/hq ADR 0131,
|
||||
// 04-ISSUES/170), so it is named here rather than left to be found.
|
||||
@@ -835,6 +866,15 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
return held, problems
|
||||
}
|
||||
|
||||
// canonicalSeat is the seat a claimed name refers to, by its current name: itself for a name the mesh
|
||||
// does not know (a module's own seat), its seat's name for a former one.
|
||||
func canonicalSeat(name string) string {
|
||||
if s, known := SeatNamed(name); known {
|
||||
return s.Name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// checkResources refuses two modules writing the same thing.
|
||||
//
|
||||
// This costs no manifest field: the mesh already holds every resource of every module, so two
|
||||
@@ -912,6 +952,23 @@ func checkResources(modules []Manifest) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A file the mesh renders for a module is that module's path too** (novox/hq ADR 0223). The
|
||||
// machine's resolver file is a fact the uplink's holder asks for, and a second module asking
|
||||
// for it, or declaring it, would have the host write one path twice in every apply, the last
|
||||
// one winning. A fact under the operator's home is placed per account and compared nowhere.
|
||||
for _, name := range sortedFacts(m.Facts) {
|
||||
fact := m.Facts[name]
|
||||
if fact.Home || !strings.HasPrefix(fact.Path, "/") {
|
||||
continue
|
||||
}
|
||||
key := "path " + fact.Path
|
||||
if other, taken := owner[key]; taken && other != m.Module {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both declare the path %q", other, m.Module, fact.Path))
|
||||
}
|
||||
owner[key] = m.Module
|
||||
ownedPath[fact.Path] = m.Module
|
||||
}
|
||||
}
|
||||
|
||||
// An accessed path is the operator's, so no module may declare it as one of its own
|
||||
@@ -1156,6 +1213,18 @@ func answeredElsewhere(want string, node Node, world World, brokered map[string]
|
||||
if c, pinned := world.Pinned[want]; pinned {
|
||||
return c.Node != node.Name
|
||||
}
|
||||
// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
|
||||
// another machine holds it too, and the first holder in the providers' order may be that one; a
|
||||
// holder is still where this machine's own requirement is answered, so its resolver file lists
|
||||
// itself first.
|
||||
if seat, delivered := SeatDelivering(want); delivered {
|
||||
for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
|
||||
if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
|
||||
h.Node == node.Name {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, held := HolderAmong(want, world.Offered[want], world.Held)
|
||||
return held && holder.Node != node.Name
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0223 part 2: /etc/resolv.conf belongs to the module holding node-uplink. The seat that
|
||||
// wrote it, node-resolver-config, and ADR 0220's dependency of it on the uplink retire: one owner for
|
||||
// the file, and it is the program that would otherwise rewrite it.
|
||||
|
||||
func TestTheResolverConfigSeatIsGone(t *testing.T) {
|
||||
if _, known := SeatNamed("node-resolver-config"); known {
|
||||
t.Error("node-resolver-config is still in the mesh's set; the uplink's holder writes the resolver file")
|
||||
}
|
||||
// An uplink's holder needs no seat beside it for the file: it is its own.
|
||||
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
|
||||
t.Errorf("an uplink holder with nothing declared depends on %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
|
||||
// uplink and writes the resolver file.
|
||||
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
|
||||
cat := map[string]Manifest{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
cat[m.Module] = m
|
||||
}
|
||||
if got := PossibleHolders(cat, "node-uplink"); !reflect.DeepEqual(got, uplinks) {
|
||||
t.Errorf("node-uplink can be held by %v, not %v", got, uplinks)
|
||||
}
|
||||
for name, m := range cat {
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "node-resolver-config" {
|
||||
t.Errorf("%s still claims node-resolver-config", name)
|
||||
}
|
||||
}
|
||||
_, writes := m.Facts["resolvers"]
|
||||
isUplink := false
|
||||
for _, u := range uplinks {
|
||||
isUplink = isUplink || u == name
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if f.Path == "/etc/resolv.conf" && !isUplink {
|
||||
t.Errorf("%s writes /etc/resolv.conf and does not hold the uplink", name)
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["path"] == "/etc/resolv.conf" {
|
||||
t.Errorf("%s declares /etc/resolv.conf as a file of its own", name)
|
||||
}
|
||||
}
|
||||
if isUplink && !writes {
|
||||
t.Errorf("%s holds the uplink and does not write the resolver file", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
|
||||
// its upstreams, or take an address systemd-resolved holds.
|
||||
|
||||
// resolverShelf is the two resolver modules and the container runtime beside something that
|
||||
// answers `mesh-addressing`.
|
||||
// resolverShelf is the resolver, an uplink module that writes what the machine asks (novox/hq ADR
|
||||
// 0223), and the container runtime beside something that answers `mesh-addressing`.
|
||||
// The networking module that really does is composed in the controller and cannot be imported
|
||||
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
|
||||
func resolverShelf(t *testing.T) map[string]Manifest {
|
||||
@@ -24,7 +24,7 @@ func resolverShelf(t *testing.T) map[string]Manifest {
|
||||
shelf := map[string]Manifest{
|
||||
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
|
||||
}
|
||||
for _, name := range []string{"dnsmasq", "resolv-conf", "docker"} {
|
||||
for _, name := range []string{"dnsmasq", "systemd-networkd", "docker"} {
|
||||
shelf[name] = catalogueManifest(t, name)
|
||||
}
|
||||
return shelf
|
||||
@@ -84,25 +84,26 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
t.Errorf("the mesh's resolver still reads or listens on %q", never)
|
||||
}
|
||||
}
|
||||
// And the file that decides what the machine asks names the mesh's resolver first, by address,
|
||||
// and a public one second, asked only when the first is silent (ADR 0196).
|
||||
var resolv string
|
||||
for _, r := range catalogueManifest(t, "resolv-conf").Resources {
|
||||
if r["path"] == "/etc/resolv.conf" {
|
||||
resolv, _ = r["content"].(string)
|
||||
// And the file that decides what the machine asks names every one of the mesh's resolvers, by
|
||||
// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
|
||||
// every listed server at once takes the first reply, and a public "no such name" for a mesh name
|
||||
// won it. Written by every uplink module, since the uplink's holder owns the file.
|
||||
for _, uplink := range uplinks {
|
||||
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
|
||||
if !ok || fact.Path != "/etc/resolv.conf" {
|
||||
t.Fatalf("%s's resolver file is not rendered from the roster: %+v", uplink, fact)
|
||||
}
|
||||
}
|
||||
var nameservers []string
|
||||
for _, line := range strings.Split(resolv, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
|
||||
if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
|
||||
t.Errorf("%s's resolv.conf does not list every holder of the mesh's resolver:\n%s", uplink, fact.Template)
|
||||
}
|
||||
for _, line := range strings.Split(fact.Template, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
|
||||
t.Errorf("%s's resolv.conf names a resolver of its own beside the mesh's: %s", uplink, line)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
|
||||
t.Errorf("%s: a silent resolver is not passed over after one short wait:\n%s", uplink, fact.Template)
|
||||
}
|
||||
}
|
||||
if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
|
||||
t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
|
||||
}
|
||||
if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
|
||||
t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,9 +112,10 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
// that file, the machine pointed at the resolver by address, and the runtime given no resolver of
|
||||
// its own but kept running across a restart (ADR 0196).
|
||||
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "docker"},
|
||||
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "systemd-networkd", "docker"},
|
||||
Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
|
||||
"package-manager": true, "service-manager": true, "privileged": true}}, World{})
|
||||
"package-manager": true, "service-manager": true, "privileged": true,
|
||||
"uplink-systemd-networkd": true}}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -126,6 +128,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
// happen to be the same map, since nothing routed is part of it.
|
||||
Names: twoMachines, Machines: twoMachines, Suffix: "internal",
|
||||
Zones: []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
|
||||
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
|
||||
})
|
||||
@@ -170,7 +173,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the resolver still writes the runtime's dns: %v", ids["dnsmasq.runtime-dns"])
|
||||
}
|
||||
for id := range ids {
|
||||
if strings.HasPrefix(id, "resolv-conf.runtime") {
|
||||
if strings.HasPrefix(id, "systemd-networkd.runtime") {
|
||||
t.Errorf("what the machine asks still writes the runtime's file: %s", id)
|
||||
}
|
||||
}
|
||||
@@ -205,29 +208,40 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
|
||||
}
|
||||
|
||||
resolv := ids["resolv-conf.resolv"]
|
||||
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
|
||||
t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
|
||||
resolv := ids["systemd-networkd.fact-resolvers"]
|
||||
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
|
||||
t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
|
||||
}
|
||||
}
|
||||
|
||||
// Two modules deciding what a machine asks are refused on one machine, as before — the claim
|
||||
// exists so they never take turns overwriting each other.
|
||||
//
|
||||
// **The second is made up.** The catalogue's only other claimant, a systemd-resolved split-DNS
|
||||
// module, was retired with the choice against a stub on every node (novox/hq ADR 0196, ADR 0220);
|
||||
// what is under test is the claim, so any second module claiming it will do.
|
||||
// Two modules deciding what a machine asks are refused on one machine, as before — now that the file
|
||||
// is the uplink's (novox/hq ADR 0223), by two things: a machine runs one network manager, and no other
|
||||
// module may write the resolver file beside the uplink's, as a file or as a rendered fact.
|
||||
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
|
||||
shelf := resolverShelf(t)
|
||||
shelf["other-resolver-config"] = Manifest{Module: "other-resolver-config", Version: "1",
|
||||
Claims: []Claim{{Name: "node-resolver-config", Scope: ScopeNode}}}
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "resolv-conf", "other-resolver-config"},
|
||||
Node{Name: "anchor", At: "anchor.internal"}, World{})
|
||||
if err == nil {
|
||||
t.Fatal("resolv-conf and a second module deciding what the machine asks were both assigned to one machine")
|
||||
shelf["networkmanager"] = catalogueManifest(t, "networkmanager")
|
||||
node := Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
|
||||
"package-manager": true, "service-manager": true,
|
||||
"uplink-systemd-networkd": true, "uplink-networkmanager": true}}
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "networkmanager"}, node, World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Fatalf("two network managers were both assigned to one machine: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "node-resolver-config") {
|
||||
t.Fatalf("the refusal does not say what was claimed: %v", err)
|
||||
|
||||
// The second is made up: what is under test is that the resolver file has one owner, so any
|
||||
// module asking the mesh to render it — or declaring it — will do.
|
||||
for name, m := range map[string]Manifest{
|
||||
"a-rendered-one": {Module: "a-rendered-one", Version: "1",
|
||||
Facts: map[string]RosterFile{"mine": {Path: "/etc/resolv.conf", Template: "nameserver 10.42.0.1\n"}}},
|
||||
"a-declared-one": {Module: "a-declared-one", Version: "1", Resources: []map[string]any{
|
||||
{"id": "mine", "type": "file", "path": "/etc/resolv.conf", "content": "nameserver 10.42.0.1\n"}}},
|
||||
} {
|
||||
shelf := resolverShelf(t)
|
||||
shelf[name] = m
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", name}, node, World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "/etc/resolv.conf") {
|
||||
t.Errorf("%s wrote the resolver file beside the uplink's: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0220: what a machine asks for names needs the uplink held beside it.
|
||||
//
|
||||
// resolv.conf is the mesh's only while the program managing the machine's network is told to leave
|
||||
// it alone, and the uplink's holder is what tells it (ADR 0117). Without one, the first connectivity
|
||||
// change rewrites the file — so the dependency is checked at assignment, by the same mechanism as a
|
||||
// service's on the service manager (ADR 0207), derived from the claim and never stated in a manifest.
|
||||
|
||||
// resolverAndUplinks is a resolver-config holder and two uplink holders, with no resources of their
|
||||
// own so that nothing but the seats is judged.
|
||||
func resolverAndUplinks() map[string]Manifest {
|
||||
return shelf(
|
||||
mod("resolv-conf", nil, nil, nil, Claim{Name: "node-resolver-config"}),
|
||||
mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"}),
|
||||
mod("systemd-networkd", nil, nil, nil, Claim{Name: "node-uplink"}),
|
||||
)
|
||||
}
|
||||
|
||||
func TestTheResolverConfigSeatNeedsTheUplink(t *testing.T) {
|
||||
s, known := SeatNamed("node-resolver-config")
|
||||
if !known {
|
||||
t.Fatal("node-resolver-config is not in the mesh's set")
|
||||
}
|
||||
if !reflect.DeepEqual(s.Needs, []string{"node-uplink"}) {
|
||||
t.Errorf("node-resolver-config needs %v, want [node-uplink] (ADR 0220)", s.Needs)
|
||||
}
|
||||
// Derived from the claim: a module claiming the seat depends on the uplink with nothing written.
|
||||
got := DependsOn(mod("anything", nil, nil, nil, Claim{Name: "node-resolver-config"}))
|
||||
if !reflect.DeepEqual(got, []string{"node-uplink"}) {
|
||||
t.Errorf("a module claiming node-resolver-config depends on %v, want [node-uplink]", got)
|
||||
}
|
||||
// And the uplink's holders need nothing of the kind: the dependency runs one way.
|
||||
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
|
||||
t.Errorf("an uplink holder depends on %v; it needs no seat beside it", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What the store loads has no column for it, so the compiled value survives a load.
|
||||
func TestTheNeedSurvivesTheStoresRows(t *testing.T) {
|
||||
defer UseSeats(DefaultSeats())
|
||||
var rows []Seat
|
||||
for _, s := range DefaultSeats() {
|
||||
rows = append(rows, Seat{Name: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision})
|
||||
}
|
||||
UseSeats(rows)
|
||||
if s, _ := SeatNamed("node-resolver-config"); !reflect.DeepEqual(s.Needs, []string{"node-uplink"}) {
|
||||
t.Errorf("after loading the store's rows node-resolver-config needs %v", s.Needs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssigningTheResolverConfigWithoutAnUplinkIsRefused(t *testing.T) {
|
||||
cat := resolverAndUplinks()
|
||||
_, err := AssignRefusal(cat, "laptop", nil, []string{"resolv-conf"})
|
||||
var refusal *Refusal
|
||||
if !errors.As(err, &refusal) {
|
||||
t.Fatalf("resolv-conf was assigned to a machine nothing manages the network of: %v", err)
|
||||
}
|
||||
for _, want := range []string{"resolv-conf on laptop depends on node-uplink", "networkmanager", "systemd-networkd"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q:\n%s", want, err)
|
||||
}
|
||||
}
|
||||
// Beside a holder, or together with one in one act, it is let through.
|
||||
if _, err := AssignRefusal(cat, "laptop", []string{"networkmanager"}, []string{"resolv-conf"}); err != nil {
|
||||
t.Errorf("resolv-conf beside networkmanager was refused: %v", err)
|
||||
}
|
||||
if _, err := AssignRefusal(cat, "anchor", nil, []string{"systemd-networkd", "resolv-conf"}); err != nil {
|
||||
t.Errorf("resolv-conf assigned with systemd-networkd was refused: %v", err)
|
||||
}
|
||||
// And a composition without one is refused once the switch is on.
|
||||
if _, err := Resolve(cat, []string{"resolv-conf"}, workstation(), World{}); err == nil ||
|
||||
!strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Errorf("a node with resolv-conf and no uplink composed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnassigningTheUplinkUnderTheResolverConfigIsRefused(t *testing.T) {
|
||||
cat := resolverAndUplinks()
|
||||
err := UnassignRefusal(cat, "laptop", []string{"networkmanager", "resolv-conf"}, []string{"networkmanager"})
|
||||
if err == nil || !strings.Contains(err.Error(), "resolv-conf") || !strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Errorf("taking the uplink from under resolv-conf gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is: the module that writes resolv.conf depends on the uplink, and every manager
|
||||
// the mesh knows can meet it — so the refusal always has a remedy to name.
|
||||
func TestTheCataloguesResolverConfigHasUplinkHoldersToName(t *testing.T) {
|
||||
cat := map[string]Manifest{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
cat[m.Module] = m
|
||||
}
|
||||
deps := DependsOn(cat["resolv-conf"])
|
||||
found := false
|
||||
for _, d := range deps {
|
||||
found = found || d == "node-uplink"
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("the catalogue's resolv-conf depends on %v, not on node-uplink", deps)
|
||||
}
|
||||
holders := PossibleHolders(cat, "node-uplink")
|
||||
for _, want := range []string{"dhcpcd", "networkmanager", "systemd-networkd"} {
|
||||
in := false
|
||||
for _, h := range holders {
|
||||
in = in || h == want
|
||||
}
|
||||
if !in {
|
||||
t.Errorf("%s does not hold node-uplink in the catalogue; holders: %v", want, holders)
|
||||
}
|
||||
}
|
||||
if got := PossibleHolders(cat, "node-resolver-config"); !reflect.DeepEqual(got, []string{"resolv-conf"}) {
|
||||
t.Errorf("node-resolver-config can be held by %v; resolv-conf alone since ADR 0220", got)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,12 @@ type rosterView struct {
|
||||
// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
|
||||
// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
|
||||
Zones []rosterZone
|
||||
// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
|
||||
// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
|
||||
// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
|
||||
// one mesh on one machine are one file. A template reads one seat's with
|
||||
// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
|
||||
Holders map[string][]rosterEntry
|
||||
}
|
||||
|
||||
// rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
|
||||
@@ -96,21 +102,25 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
|
||||
// FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
|
||||
func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
|
||||
zones []ZoneAt) ([]map[string]any, error) {
|
||||
return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
|
||||
Zones: zones})
|
||||
}
|
||||
|
||||
// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
|
||||
// machine: its machines, zones and the holders of each replicated seat.
|
||||
func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
|
||||
if len(m.Facts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
names := make([]string, 0, len(m.Facts))
|
||||
for name := range m.Facts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
names := sortedFacts(m.Facts)
|
||||
|
||||
view := rosterView{
|
||||
Node: r.Node,
|
||||
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
|
||||
Names: entriesFrom(every, accounts, suffix),
|
||||
Machines: entriesFrom(machines, accounts, suffix),
|
||||
Zones: zonesFrom(zones),
|
||||
Suffix: strings.TrimPrefix(suffixOr(with.Suffix), "."),
|
||||
Names: entriesFrom(with.Names, with.Accounts, with.Suffix),
|
||||
Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
|
||||
Zones: zonesFrom(with.Zones),
|
||||
Holders: holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
|
||||
}
|
||||
|
||||
out := make([]map[string]any, 0, len(names))
|
||||
@@ -250,3 +260,28 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
|
||||
return out
|
||||
}
|
||||
|
||||
// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
|
||||
// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
|
||||
// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
|
||||
func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
|
||||
out := make(map[string][]rosterEntry, len(holders))
|
||||
for seat, at := range holders {
|
||||
entries := entriesFrom(at, accounts, suffix)
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
return entries[i].Name == node && entries[j].Name != node
|
||||
})
|
||||
out[seat] = entries
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedFacts is a module's fact names in order, so what is said about them is said the same way twice.
|
||||
func sortedFacts(facts map[string]RosterFile) []string {
|
||||
out := make([]string, 0, len(facts))
|
||||
for name := range facts {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -6,9 +6,10 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A module depends on the node seats that apply its resources (novox/hq ADR 0207), on the
|
||||
// seats it contributes to (novox/hq ADR 0210), and on the seats a seat it holds needs beside it
|
||||
// (novox/hq ADR 0220).
|
||||
// A module depends on the node seats that apply its resources (novox/hq ADR 0207) and on the
|
||||
// seats it contributes to (novox/hq ADR 0210). A third source — what a seat it holds needs beside it
|
||||
// (novox/hq ADR 0220) — had one user, node-resolver-config needing node-uplink, and went with that
|
||||
// seat when the resolver file became the uplink's own (novox/hq ADR 0223).
|
||||
//
|
||||
// Some of what a module declares is applied through software on the machine that is itself a
|
||||
// module: a service through the service manager, a package through the package manager, a container
|
||||
@@ -94,9 +95,6 @@ func DependsOn(m Manifest) []string {
|
||||
for _, seat := range contributedTo(m) {
|
||||
seen[seat] = true
|
||||
}
|
||||
for _, seat := range neededBesideClaims(m) {
|
||||
seen[seat] = true
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for s := range seen {
|
||||
out = append(out, s)
|
||||
@@ -128,20 +126,6 @@ func contributedTo(m Manifest) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// neededBesideClaims is every seat a seat the module claims at node scope needs held on the same
|
||||
// node (novox/hq ADR 0220): the holder of node-resolver-config is only right while node-uplink's
|
||||
// holder keeps the network manager off resolv.conf. Derived from the claim, as a resource's seat is
|
||||
// derived from its type, so a module that claims the seat cannot leave the dependency out.
|
||||
func neededBesideClaims(m Manifest) []string {
|
||||
var out []string
|
||||
for _, name := range nodeSeatsClaimed(m) {
|
||||
if s, known := SeatNamed(name); known {
|
||||
out = append(out, s.Needs...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// claimsSeat is whether a module claims a node seat, by its current name or one it used to have
|
||||
// (ADR 0122), so a rename leaves the dependency met.
|
||||
func claimsSeat(m Manifest, seat string) bool {
|
||||
|
||||
+45
-39
@@ -21,8 +21,16 @@ import (
|
||||
type Seat struct {
|
||||
// Name is what a manifest claims.
|
||||
Name string
|
||||
// Scope is where there may be only one holder.
|
||||
// Scope is where there may be only one holder — unless the seat is Replicated.
|
||||
Scope string
|
||||
// Replicated says a mesh seat may be held on several machines at once, each holder answering the
|
||||
// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
|
||||
// machine's resolver file lists two that give one answer. Each holder is on record, added by an
|
||||
// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
|
||||
// record are refused exactly as for any mesh seat. One per machine still — two modules on one
|
||||
// node claiming it are refused. Compiled, never stored, like Receives: it is the mesh's definition of
|
||||
// the role, and the store's rows carry no column for it.
|
||||
Replicated bool
|
||||
// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
|
||||
// provision may only be held by a module providing it at the seat's scope, and its holder is
|
||||
// what a requirement for that provision resolves to when several modules provide it.
|
||||
@@ -45,13 +53,6 @@ type Seat struct {
|
||||
// ${contribution:<seat>:<kind>}. Compiled, never stored: like the protocol, it is the mesh's
|
||||
// definition of the role, and the store's rows carry no column for it.
|
||||
Receives []Receivable
|
||||
// Needs is every node seat this seat's holder needs held on its own node (novox/hq ADR 0220): a
|
||||
// role whose holder is only right while another role is filled beside it. A module claiming this
|
||||
// seat depends on each, exactly as a module declaring a service depends on the service manager
|
||||
// (ADR 0207) — derived from the claim, never written in a manifest, and judged over the node's
|
||||
// whole set of assignments. Compiled, never stored, like Receives: it is the mesh's definition of
|
||||
// the role, and the store's rows carry no column for it.
|
||||
Needs []string
|
||||
// Decision is the record that made it a seat.
|
||||
Decision string
|
||||
}
|
||||
@@ -140,14 +141,20 @@ var defaultSeats = append([]Seat{
|
||||
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
|
||||
{Name: "mesh-build-machine", Scope: ScopeMesh,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
|
||||
// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
|
||||
// place, and every node and container asks it first. Delivers what a machine's resolver
|
||||
// configuration requires, so that requirement resolves to the holder wherever it is placed.
|
||||
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
|
||||
// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
|
||||
// own lines and keeps every other line as the operator's, changed through these three verbs on that
|
||||
// machine alone. The controller holds none of it.
|
||||
{Name: "node-hosts-file", Scope: ScopeNode, Decision: "novox/hq ADR 0199",
|
||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||
// than one machine, each answering the same names from the same roster, and every machine's
|
||||
// resolver file lists every holder — its own first — and no public resolver, so whichever answers
|
||||
// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
|
||||
// requirement resolves to a holder wherever they are placed.
|
||||
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
|
||||
Decision: "novox/hq ADR 0194, ADR 0223"},
|
||||
// **A machine's names are one module's** (novox/hq ADR 0199, ADR 0223): its holder writes
|
||||
// /etc/hostname and the machine's own lines in /etc/hosts, and keeps every other line of the hosts
|
||||
// file as the operator's, changed through these three verbs on that machine alone. The controller
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
@@ -248,18 +255,12 @@ var defaultSeats = append([]Seat{
|
||||
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||
// model change, not a rename, so it stays until that is built.
|
||||
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||
// What a machine asks for names (novox/hq ADR 0121, ADR 0196): its holder writes resolv.conf.
|
||||
// **And it needs the uplink held beside it** (novox/hq ADR 0220): resolv.conf stays the mesh's
|
||||
// only while the program managing the machine's network is told to keep its hands off it, and
|
||||
// that is what the uplink's holder says (ADR 0117). Without one, the first connectivity change
|
||||
// rewrites the file and every surface of the mesh still reads green — so it is refused at
|
||||
// assignment instead.
|
||||
{Name: "node-resolver-config", Scope: ScopeNode, Decision: "novox/hq ADR 0121, ADR 0220",
|
||||
Needs: []string{"node-uplink"}},
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder only
|
||||
// keeps the manager and the mesh from contradicting each other — the resolver file left to the
|
||||
// mesh, the private network's interface left alone — and never declares a link, an address or
|
||||
// a wireless network, because the link is the only channel a fix could arrive on. A seat
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder keeps the
|
||||
// manager and the mesh from contradicting each other — the private network's interface left
|
||||
// alone — and writes the machine's resolver file itself, because the manager is what would
|
||||
// otherwise rewrite it (novox/hq ADR 0223, which retired node-resolver-config into this seat).
|
||||
// It never declares a link, an address or a wireless network, because the link is the only
|
||||
// channel a fix could arrive on. A seat
|
||||
// rather than a condition in the resolver's module, so a machine running two managers is
|
||||
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
|
||||
@@ -314,11 +315,11 @@ func UseSeats(s []Seat) {
|
||||
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
|
||||
}
|
||||
}
|
||||
// What a seat receives and what its holder needs are never stored (novox/hq ADR 0212, ADR
|
||||
// 0220), so they are always the compiled ones.
|
||||
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
|
||||
// 0223), so they are always the compiled ones.
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Needs = d.Needs
|
||||
row.Replicated = d.Replicated
|
||||
}
|
||||
merged = append(merged, row)
|
||||
}
|
||||
@@ -498,19 +499,24 @@ func seatNames() string {
|
||||
// two modules on one node could both provide a provision, and only the one holding the seat
|
||||
// answers for it. Nothing when no seat delivers the provision, when nobody holds
|
||||
// it, or when the holder is not among the providers offered.
|
||||
//
|
||||
// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
|
||||
// several, and the answer must not depend on the order the mesh happened to resolve its machines
|
||||
// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
|
||||
// one holder gets the same answer as before.
|
||||
func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
|
||||
seat, delivered := SeatDelivering(provision)
|
||||
if !delivered {
|
||||
return Provider{}, false
|
||||
}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
|
||||
// former name still matches it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||
continue
|
||||
}
|
||||
for _, p := range providers {
|
||||
for _, p := range providers {
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
// seat's former name still matches it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||
continue
|
||||
}
|
||||
if p.Node == h.Node && p.Module == h.Module {
|
||||
return p, true
|
||||
}
|
||||
|
||||
@@ -46,16 +46,18 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Thirty-seven since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
// Thirty-six since node-resolver-config retired into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
|
||||
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
|
||||
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199, now
|
||||
// node-hostname); thirty-four
|
||||
// with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the
|
||||
// graphical session's eleven (ADR 0208); twenty-one with node-package-manager and
|
||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
if len(Seats()) != 37 {
|
||||
t.Errorf("the mesh defines %d seats rather than 37; the set is closed, so a change here is "+
|
||||
if len(Seats()) != 36 {
|
||||
t.Errorf("the mesh defines %d seats rather than 36; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
|
||||
// anchor and on the home server, each answering the same names; every machine's resolver file lists
|
||||
// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
|
||||
// resolver then a public one, and musl asks both at once and takes the first reply: from the home
|
||||
// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
|
||||
// The file is written by the module holding the machine's uplink (ADR 0223 part 2).
|
||||
|
||||
// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
|
||||
var resolverMachines = map[string]string{
|
||||
"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
|
||||
|
||||
// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
|
||||
var bothResolvers = []Held{
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
|
||||
}
|
||||
|
||||
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
|
||||
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
|
||||
var uplinks = []string{"dhcpcd", "networkmanager", "systemd-networkd"}
|
||||
|
||||
// managing is a machine as each uplink module needs it: able to install, run a service and run the
|
||||
// manager that module is for.
|
||||
func managing(node string) Node {
|
||||
caps := map[string]bool{"package-manager": true, "service-manager": true}
|
||||
for _, u := range uplinks {
|
||||
caps["uplink-"+u] = true
|
||||
}
|
||||
return Node{Name: node, At: node + ".internal", Capabilities: caps}
|
||||
}
|
||||
|
||||
// twoResolverShelf is the resolver, the uplink modules that write what a machine asks, and a stand-in
|
||||
// answering `mesh-addressing`.
|
||||
func twoResolverShelf(t *testing.T) map[string]Manifest {
|
||||
t.Helper()
|
||||
out := map[string]Manifest{
|
||||
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
|
||||
"dnsmasq": catalogueManifest(t, "dnsmasq"),
|
||||
}
|
||||
for _, u := range uplinks {
|
||||
out[u] = catalogueManifest(t, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
|
||||
// and offer, and both holders on record.
|
||||
func worldWithout(node string) World {
|
||||
w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
|
||||
for _, h := range bothResolvers {
|
||||
if h.Node == node {
|
||||
continue
|
||||
}
|
||||
w.Held = append(w.Held, h)
|
||||
w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
|
||||
Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
|
||||
// lists, in order, and the file. The file is the uplink's — `uplink` is one of the assigned modules —
|
||||
// and nothing else on the machine declares that path.
|
||||
func resolvConfOn(t *testing.T, node, uplink string, assigned []string) ([]string, string) {
|
||||
t.Helper()
|
||||
got, err := Resolve(twoResolverShelf(t), assigned, managing(node), worldWithout(node))
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s does not resolve with two resolvers on record: %v", node, uplink, err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{
|
||||
Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
|
||||
Holders: map[string]map[string]string{"mesh-dns-resolver": {
|
||||
"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s does not compose: %v", node, uplink, err)
|
||||
}
|
||||
var file map[string]any
|
||||
for _, r := range out {
|
||||
if r["path"] != "/etc/resolv.conf" {
|
||||
continue
|
||||
}
|
||||
if file != nil {
|
||||
t.Fatalf("%s declares /etc/resolv.conf twice: %v and %v", node, file["id"], r["id"])
|
||||
}
|
||||
file = r
|
||||
}
|
||||
if file == nil || file["id"] != uplink+".fact-resolvers" {
|
||||
t.Fatalf("%s's resolver file is not %s's: %v", node, uplink, file)
|
||||
}
|
||||
content, _ := file["content"].(string)
|
||||
var servers []string
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
servers = append(servers, strings.TrimPrefix(line, "nameserver "))
|
||||
}
|
||||
}
|
||||
return servers, content
|
||||
}
|
||||
|
||||
// Per uplink module: each writes a resolver file listing both holders, the machine's own first on a
|
||||
// holder, and only the holders on a machine that is none.
|
||||
func TestEveryUplinkListsBothResolversItsOwnFirst(t *testing.T) {
|
||||
for _, uplink := range uplinks {
|
||||
for node, want := range map[string][]string{
|
||||
"anchor": {"10.42.0.1", "10.42.0.3"},
|
||||
"home": {"10.42.0.3", "10.42.0.1"},
|
||||
"laptop": {"10.42.0.1", "10.42.0.3"},
|
||||
} {
|
||||
assigned := []string{uplink}
|
||||
if node != "laptop" {
|
||||
assigned = append(assigned, "dnsmasq")
|
||||
}
|
||||
servers, content := resolvConfOn(t, node, uplink, assigned)
|
||||
if strings.Join(servers, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("%s on %s lists %v; want %v\n%s", uplink, node, servers, want, content)
|
||||
}
|
||||
for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
|
||||
if strings.Contains(content, public) {
|
||||
t.Errorf("%s on %s lists a public resolver beside the mesh's (ADR 0223):\n%s", uplink, node, content)
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(content, "\noptions timeout:1 attempts:2 edns0\n") {
|
||||
t.Errorf("%s on %s does not end with two short attempts:\n%s", uplink, node, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One file, whichever manager the machine runs: the three modules carry the same template, so a
|
||||
// machine changing its manager changes nothing in what it asks, and a fix made to one is made to all.
|
||||
func TestEveryUplinkWritesTheSameResolverFile(t *testing.T) {
|
||||
var first, firstOf string
|
||||
for _, uplink := range uplinks {
|
||||
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
|
||||
if !ok || fact.Path != "/etc/resolv.conf" || fact.Shared || fact.Home {
|
||||
t.Fatalf("%s does not write the machine's resolver file whole: %+v", uplink, fact)
|
||||
}
|
||||
if first == "" {
|
||||
first, firstOf = fact.Template, uplink
|
||||
continue
|
||||
}
|
||||
if fact.Template != first {
|
||||
t.Errorf("%s's resolver file differs from %s's; the three are kept identical", uplink, firstOf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The requirement stays with what writes the file (ADR 0223 part 1): a machine is refused when nothing
|
||||
// in the mesh resolves, rather than given a file listing nothing.
|
||||
func TestEveryUplinkRequiresTheMeshsResolver(t *testing.T) {
|
||||
for _, uplink := range uplinks {
|
||||
found := false
|
||||
for _, r := range catalogueManifest(t, uplink).Requires {
|
||||
found = found || r == "wildcard-resolution"
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s writes the resolver file and does not require wildcard-resolution", uplink)
|
||||
}
|
||||
}
|
||||
_, err := Resolve(twoResolverShelf(t), []string{"networkmanager"}, managing("laptop"), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "wildcard-resolution") {
|
||||
t.Errorf("an uplink was composed on a mesh with no resolver: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
|
||||
func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
|
||||
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "networkmanager"},
|
||||
managing("home"), worldWithout("home"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range got.Needs {
|
||||
if n.Name == "wildcard-resolution" && n.From != "home" {
|
||||
t.Errorf("the home server's uplink is bound to %s; it holds the seat itself", n.From)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat held once is still held once: a second claimant on another machine is refused while
|
||||
// nothing is on record, and a store recording two holders of it is refused, naming the seat.
|
||||
func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
|
||||
store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
|
||||
other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
|
||||
if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
|
||||
!strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Errorf("a second claimant of a seat held once was not refused: %v", err)
|
||||
}
|
||||
here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
|
||||
_, err := Resolve(store, []string{"postgres"}, workstation(),
|
||||
World{Held: []Held{other}, Holdings: []Held{other, here}})
|
||||
if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
|
||||
t.Errorf("two holders on record for a seat held once were not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
|
||||
// any mesh seat, and each holder is added by an act.
|
||||
func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
|
||||
w := worldWithout("home")
|
||||
w.Holdings = nil
|
||||
_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
|
||||
if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
|
||||
t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
|
||||
func TestOnlyTheResolverIsReplicated(t *testing.T) {
|
||||
for _, s := range Seats() {
|
||||
if s.Replicated != (s.Name == "mesh-dns-resolver") {
|
||||
t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
|
||||
}
|
||||
}
|
||||
UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
|
||||
defer UseSeats(DefaultSeats())
|
||||
if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
|
||||
t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
|
||||
}
|
||||
}
|
||||
|
||||
// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
|
||||
func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
|
||||
providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
|
||||
for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
|
||||
if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
|
||||
t.Errorf("held in order %v answered %v", held, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
|
||||
// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
|
||||
// musl reads that as final.
|
||||
func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
|
||||
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
|
||||
World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
|
||||
records := map[string]int{}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.HasPrefix(line, "host-record=") {
|
||||
records[strings.TrimPrefix(line, "host-record=")]++
|
||||
}
|
||||
}
|
||||
for name, at := range resolverMachines {
|
||||
if records[name+","+at] != 1 {
|
||||
t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
|
||||
}
|
||||
}
|
||||
if len(records) != len(resolverMachines) {
|
||||
t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
|
||||
}
|
||||
}
|
||||
@@ -124,7 +124,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
|
||||
d := broker.Declared{
|
||||
Module: m.Module,
|
||||
Emits: m.Emits,
|
||||
Emits: m.EmitsAll(),
|
||||
Consumes: fromModules,
|
||||
Watches: watches,
|
||||
// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the
|
||||
|
||||
@@ -97,3 +97,57 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
|
||||
t.Fatalf("a re-told membership did not replace the first: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
|
||||
// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
|
||||
// takes only its own row.
|
||||
func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
|
||||
resolver := catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
|
||||
inv, ctx := aMeshWith(t, resolver)
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"anchor", "home"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
held, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
|
||||
t.Fatalf("the two holders are not both on record, once each: %+v", held)
|
||||
}
|
||||
if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
|
||||
t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
|
||||
t.Fatalf("a handover left other holders on record: %+v", held)
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
-- A replicated seat has several holders on record (novox/hq ADR 0223).
|
||||
--
|
||||
-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
|
||||
-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
|
||||
-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
|
||||
-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
|
||||
--
|
||||
-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
|
||||
-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
|
||||
-- judged by the controller before a row is added, and a store holding two for any other seat is
|
||||
-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
|
||||
-- key as it stands.
|
||||
alter table seat_holding drop constraint seat_holding_pkey;
|
||||
alter table seat_holding add primary key (seat, node, module);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- What a machine asks for names is the uplink's holder's to write (novox/hq ADR 0223, retiring what
|
||||
-- ADR 0121 and ADR 0220 decided for node-resolver-config).
|
||||
--
|
||||
-- `/etc/resolv.conf` is written by the module holding `node-uplink` — the program that would otherwise
|
||||
-- rewrite it — so the seat whose holder wrote it, and its need of the uplink beside it, go. Its only
|
||||
-- claimant, `resolv-conf`, declared nothing for one release while every machine handed the file to its
|
||||
-- uplink module in one apply, and was then unassigned everywhere and forgotten before this runs.
|
||||
--
|
||||
-- **The compiled defaults no longer carry it, and that alone would not remove it**: seeding adds a
|
||||
-- seat a release ships and never takes one away (ADR 0122), as 0060 found for the per-node resolver.
|
||||
-- A holding on record goes with it by cascade; a node seat has none. No alias is kept: nothing was
|
||||
-- renamed, and a manifest still claiming the old name should be refused at registration, naming it.
|
||||
--
|
||||
-- Numbered after 0063 (node-hosts-file renamed to node-hostname), which is expected to merge first;
|
||||
-- if this one lands first, the two are renumbered so the order they merge in is the order they run.
|
||||
delete from seat_alias where seat = 'node-resolver-config' or alias = 'node-resolver-config';
|
||||
delete from seat where name = 'node-resolver-config';
|
||||
@@ -0,0 +1,23 @@
|
||||
-- A machine's names are one seat's (novox/hq ADR 0223 part 3): `node-hosts-file` is renamed
|
||||
-- `node-hostname`, whose holder writes /etc/hostname beside the machine's own lines in /etc/hosts.
|
||||
--
|
||||
-- A rename is a database update (ADR 0122): the row keeps its verbs, the former name becomes an alias
|
||||
-- that resolves to it, so a manifest registered under the old name — the `hosts` module still assigned
|
||||
-- while machines move to `hostname` — goes on holding the one seat, and two claimants of it on one
|
||||
-- machine are still refused.
|
||||
--
|
||||
-- **Both rows may exist when this runs**, as 0048 found for the artifact store: a controller whose
|
||||
-- compiled defaults carry the new name may seed it before this migration. Then the old row's holding
|
||||
-- moves to it and the old row goes; otherwise the old row is renamed. Either way the old name becomes
|
||||
-- an alias.
|
||||
update seat_holding set seat = 'node-hostname'
|
||||
where seat = 'node-hosts-file'
|
||||
and exists (select 1 from seat where name = 'node-hostname');
|
||||
delete from seat
|
||||
where name = 'node-hosts-file'
|
||||
and exists (select 1 from seat where name = 'node-hostname');
|
||||
update seat set name = 'node-hostname', decided = 'novox/hq ADR 0199, ADR 0223'
|
||||
where name = 'node-hosts-file';
|
||||
insert into seat_alias (alias, seat) values ('node-hosts-file', 'node-hostname')
|
||||
on conflict (alias) do update set seat = excluded.seat;
|
||||
update seat_alias set seat = 'node-hostname' where seat = 'node-hosts-file';
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
-- A provider says which consumer it keeps failing (novox/hq ADR 0224).
|
||||
--
|
||||
-- On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a day and
|
||||
-- only its journal said so (issue 179). A provider now announces a consumer it has failed for minutes
|
||||
-- without one success, and the consumer recovering; the controller keeps the newest failing word per
|
||||
-- provider module, the machine it runs on and the consumer, and removes it on recovery. `status` and
|
||||
-- `node show` read this table: a row is a problem until it is gone.
|
||||
create table provider_standing (
|
||||
module text not null,
|
||||
provider_node text not null,
|
||||
consumer text not null,
|
||||
consumer_node text not null default '',
|
||||
provision text not null default '',
|
||||
class text not null default '',
|
||||
error text not null default '',
|
||||
since timestamptz not null,
|
||||
attempts integer not null default 0,
|
||||
said_at timestamptz not null default now(),
|
||||
primary key (module, provider_node, consumer)
|
||||
);
|
||||
+37
-10
@@ -215,23 +215,50 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
|
||||
// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
|
||||
// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
|
||||
// cannot be handed to something that is not running anywhere.
|
||||
// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
|
||||
// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
|
||||
// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
|
||||
// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
|
||||
// running anywhere.
|
||||
func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
|
||||
node, err := i.NodeByName(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
|
||||
module = excluded.module, since = now()`,
|
||||
seat, scope, node.ID, module)
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx,
|
||||
`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
|
||||
seat, node.ID, module); err != nil {
|
||||
return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
|
||||
seat, scope, node.ID, module); err != nil {
|
||||
return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
|
||||
// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
|
||||
// the seat's definition is compiled, and the store holds no column for it. Recording a holder
|
||||
// already on record changes nothing.
|
||||
func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
|
||||
node, err := i.NodeByName(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat, node, module) do nothing`,
|
||||
seat, scope, node.ID, module); err != nil {
|
||||
return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -240,7 +267,7 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
|
||||
func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
|
||||
from seat_holding h join node n on n.id = h.node order by h.seat`)
|
||||
from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
|
||||
type ProviderStanding struct {
|
||||
// Module is the provider's module, and ProviderNode the machine it runs on.
|
||||
Module string `json:"module"`
|
||||
ProviderNode string `json:"provider-node"`
|
||||
// Provision is the interface it provides, e.g. `oidc-client`.
|
||||
Provision string `json:"provision"`
|
||||
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
|
||||
Consumer string `json:"consumer"`
|
||||
ConsumerNode string `json:"consumer-node"`
|
||||
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
|
||||
Class string `json:"class"`
|
||||
Error string `json:"error"`
|
||||
// Since is when the unbroken run of failures began; Attempts how many it has been.
|
||||
Since time.Time `json:"since"`
|
||||
Attempts int `json:"attempts"`
|
||||
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
|
||||
// while it lasts, so an old one is a provider that stopped saying anything.
|
||||
SaidAt time.Time `json:"said-at"`
|
||||
}
|
||||
|
||||
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
|
||||
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
|
||||
const SayAgainWithin = 30 * time.Minute
|
||||
|
||||
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
|
||||
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
|
||||
|
||||
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
|
||||
// whether a recovery removed anything.
|
||||
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
|
||||
if !failing {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
|
||||
s.Module, s.ProviderNode, s.Consumer)
|
||||
return err == nil && tag.RowsAffected() > 0, err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx, `
|
||||
insert into provider_standing
|
||||
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
||||
on conflict (module, provider_node, consumer) do update set
|
||||
consumer_node = excluded.consumer_node, provision = excluded.provision,
|
||||
class = excluded.class, error = excluded.error, since = excluded.since,
|
||||
attempts = excluded.attempts, said_at = excluded.said_at`,
|
||||
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
|
||||
return false, err
|
||||
}
|
||||
|
||||
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
|
||||
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `
|
||||
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
|
||||
from provider_standing order by since, module, consumer`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ProviderStanding
|
||||
for rows.Next() {
|
||||
var s ProviderStanding
|
||||
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
|
||||
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
|
||||
// reads.
|
||||
|
||||
// The broker spells the events itself because it cannot import the catalogue; the two agree.
|
||||
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
|
||||
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
|
||||
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
|
||||
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
|
||||
}
|
||||
}
|
||||
|
||||
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
|
||||
// events would have its announcement refused by the bus, and fail its consumers as silently as on
|
||||
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
|
||||
// nothing.
|
||||
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
|
||||
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
|
||||
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
|
||||
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
|
||||
|
||||
d := declaredFor(provider, nil)
|
||||
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
|
||||
if !slices.Contains(d.Emits, e) {
|
||||
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
|
||||
}
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
|
||||
Module: "keycloak", Emits: d.Emits})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
|
||||
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
|
||||
}
|
||||
|
||||
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
|
||||
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
|
||||
}
|
||||
// Declared by hand as well: said once.
|
||||
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
|
||||
n := 0
|
||||
for _, e := range provider.EmitsAll() {
|
||||
if e == catalogue.ProvisionerFailing {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("%v", provider.EmitsAll())
|
||||
}
|
||||
}
|
||||
|
||||
// And the controller may hear it from every provider, and only those two events.
|
||||
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
|
||||
if !slices.Contains(perms.Subscribe, want) {
|
||||
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
|
||||
}
|
||||
}
|
||||
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
|
||||
t.Fatal("the controller hears every event in the mesh")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
|
||||
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
|
||||
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
|
||||
Error: "401 invalid_grant", Since: since, Attempts: 60}
|
||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Said again: one row, the newest word.
|
||||
s.Attempts = 31000
|
||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
|
||||
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if got[0].Quiet(time.Now()) {
|
||||
t.Fatal("a standing just said reads as quiet")
|
||||
}
|
||||
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
|
||||
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
|
||||
}
|
||||
|
||||
// The same consumer from another machine's provider is its own row.
|
||||
other := s
|
||||
other.ProviderNode = "laptop"
|
||||
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleared, err := inv.KeepStanding(ctx, false, s)
|
||||
if err != nil || !cleared {
|
||||
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
|
||||
}
|
||||
cleared, err = inv.KeepStanding(ctx, false, s)
|
||||
if err != nil || cleared {
|
||||
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
|
||||
}
|
||||
got, err = inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ProviderNode != "laptop" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,9 @@ const (
|
||||
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
||||
KindSourceMoved = "source-moved"
|
||||
KindCatchUp = "catch-up"
|
||||
// KindProvisioner is a provider saying a consumer has failed for minutes, or recovered
|
||||
// (novox/hq ADR 0224).
|
||||
KindProvisioner = "provisioner"
|
||||
)
|
||||
|
||||
// Control is one thing a node or a module said, as the controller must act on it.
|
||||
@@ -54,6 +57,11 @@ type Control interface {
|
||||
// Body is the message itself — the payload alone, never the envelope.
|
||||
Body() []byte
|
||||
|
||||
// Subject is where it was published. For a module's event it names the emitter, which the bus
|
||||
// enforces (only a module may publish into its own namespace), so who said it is read from here
|
||||
// and never from the body.
|
||||
Subject() string
|
||||
|
||||
// Redelivered says the bus has handed this message over before. An enrolment cares and
|
||||
// nothing else does: one already spent is not finished a second time.
|
||||
Redelivered() bool
|
||||
|
||||
@@ -61,6 +61,7 @@ func (c *fakeInbound) retries(ctx context.Context, s *Server) {
|
||||
|
||||
type fakeControl struct {
|
||||
kind string
|
||||
subject string
|
||||
body []byte
|
||||
tag uint64
|
||||
to *settled
|
||||
@@ -71,6 +72,7 @@ type fakeControl struct {
|
||||
|
||||
func (m *fakeControl) Kind() string { return m.kind }
|
||||
func (m *fakeControl) Body() []byte { return m.body }
|
||||
func (m *fakeControl) Subject() string { return m.subject }
|
||||
func (m *fakeControl) Redelivered() bool { return m.redelivered }
|
||||
func (m *fakeControl) About(string) {}
|
||||
func (m *fakeControl) Answer(context.Context, []byte) error { return nil }
|
||||
|
||||
@@ -53,7 +53,7 @@ func Nats(js *broker.JetStream) Inbound {
|
||||
// whatever was asked for — and not at all when nothing was.
|
||||
func (n *natsInbound) Also(kind string) error {
|
||||
switch kind {
|
||||
case KindModuleMoved, KindCatchUp, KindSourceMoved:
|
||||
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner:
|
||||
n.follows[kind] = true
|
||||
return nil
|
||||
default:
|
||||
@@ -241,9 +241,30 @@ func kindOfSubject(subject string) (string, bool) {
|
||||
// runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned.
|
||||
return KindBuilt, true
|
||||
}
|
||||
if _, ok := ProvisionerEmitter(subject); ok {
|
||||
return KindProvisioner, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ProvisionerEmitter is the module a provider's standing event came from, read from its subject
|
||||
// (`mesh.mod.<module>.event.provisioner.<failing|recovered>`); false for any other subject. The
|
||||
// controller's own follow pattern, with `*` for the module, decodes too.
|
||||
func ProvisionerEmitter(subject string) (string, bool) {
|
||||
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
module, event, ok := strings.Cut(rest, ".event.")
|
||||
if !ok || module == "" || strings.Contains(module, ".") {
|
||||
return "", false
|
||||
}
|
||||
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered {
|
||||
return "", false
|
||||
}
|
||||
return module, true
|
||||
}
|
||||
|
||||
// natsControl is one message from the bus being built, as the controller reads it.
|
||||
type natsControl struct {
|
||||
kind string
|
||||
@@ -256,8 +277,9 @@ type natsControl struct {
|
||||
delivered uint64
|
||||
}
|
||||
|
||||
func (m *natsControl) Kind() string { return m.kind }
|
||||
func (m *natsControl) Body() []byte { return m.msg.Data }
|
||||
func (m *natsControl) Kind() string { return m.kind }
|
||||
func (m *natsControl) Body() []byte { return m.msg.Data }
|
||||
func (m *natsControl) Subject() string { return m.msg.Subject }
|
||||
|
||||
// Redelivered is what the server counted, not what the controller remembers. Which is the answer to
|
||||
// a question the AMQP side could only guess at across a restart: an enrolment redelivered because
|
||||
|
||||
@@ -79,6 +79,8 @@ type Server struct {
|
||||
recorder Recorder
|
||||
upgrader Upgrader
|
||||
replayer Replayer
|
||||
// standings keeps what providers say about their consumers (novox/hq ADR 0224).
|
||||
standings Standings
|
||||
|
||||
log *log.Logger
|
||||
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
||||
@@ -153,6 +155,9 @@ func (s *Server) Serve(ctx context.Context) error {
|
||||
if s.replayer != nil {
|
||||
s.log.Printf("answering %s", KindCatchUp)
|
||||
}
|
||||
if s.standings != nil {
|
||||
s.log.Printf("keeping every provider's %s", KindProvisioner)
|
||||
}
|
||||
return s.inbound.Receive(ctx, s.act)
|
||||
}
|
||||
|
||||
@@ -173,6 +178,8 @@ func (s *Server) act(ctx context.Context, m Control) {
|
||||
s.sourceMoved(ctx, m)
|
||||
case KindCatchUp:
|
||||
s.catchingUp(ctx, m)
|
||||
case KindProvisioner:
|
||||
s.provisioner(ctx, m)
|
||||
default:
|
||||
// Dropped: a message nothing understands will not be understood on the next attempt
|
||||
// either, and asking for it again would spin.
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A provider's standing (novox/hq ADR 0224).
|
||||
//
|
||||
// **A provider that keeps failing a consumer is a problem the controller reports**, not a line in a
|
||||
// journal. On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a
|
||||
// day — its admin no longer took the mesh's secret once its database was moved — and every surface
|
||||
// the mesh has called the mesh well (novox/hq issue 179). A provider now says, as an event, a
|
||||
// consumer it has failed for minutes without one success, and the consumer recovering; the
|
||||
// controller keeps the newest word per provider, machine and consumer, and `status` names each one
|
||||
// still failing.
|
||||
|
||||
// Standing is one provider's word about one consumer.
|
||||
type Standing struct {
|
||||
// Module is the emitter, read from the subject the bus let it publish on — never from the body.
|
||||
Module string `json:"-"`
|
||||
// Failing is which of the two it said: failing, or recovered.
|
||||
Failing bool `json:"-"`
|
||||
|
||||
Provider string `json:"provider"`
|
||||
ProviderNode string `json:"provider-node"`
|
||||
Consumer string `json:"consumer"`
|
||||
Node string `json:"node"`
|
||||
Class string `json:"class,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Attempts int `json:"attempts"`
|
||||
// Why is said with a recovery that is not a success: `withdrawn`, a consumer no longer asked for.
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// Standings keeps what providers say about their consumers.
|
||||
type Standings interface {
|
||||
// Stood records a provider's newest word about a consumer: a failing one kept, a recovered one
|
||||
// cleared — and says whether a recovery cleared anything, since a provider announces its first
|
||||
// success for every consumer after it starts. An error the store is away for is held and asked
|
||||
// again, like a report.
|
||||
Stood(ctx context.Context, s Standing) (cleared bool, err error)
|
||||
}
|
||||
|
||||
// Watches says where providers' standings are kept, and asks for them to be delivered.
|
||||
func (s *Server) Watches(st Standings) error {
|
||||
if err := s.inbound.Also(KindProvisioner); err != nil {
|
||||
return err
|
||||
}
|
||||
s.standings = st
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadStanding is one standing event as the controller understands it, from its subject and body.
|
||||
func ReadStanding(subject string, body []byte) (Standing, error) {
|
||||
module, ok := ProvisionerEmitter(subject)
|
||||
if !ok {
|
||||
return Standing{}, fmt.Errorf("%s is not a provider's standing", subject)
|
||||
}
|
||||
var st Standing
|
||||
if err := json.Unmarshal(body, &st); err != nil {
|
||||
return Standing{}, fmt.Errorf("%s's standing could not be read: %w", module, err)
|
||||
}
|
||||
if st.Consumer == "" {
|
||||
return Standing{}, fmt.Errorf("%s's standing named no consumer", module)
|
||||
}
|
||||
st.Module = module
|
||||
st.Failing = strings.HasSuffix(subject, "."+broker.ProvisionerFailing)
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// provisioner acts on one standing event.
|
||||
//
|
||||
// **A recovery must not be lost.** A failing standing is said again every quarter of an hour while
|
||||
// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
|
||||
// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
|
||||
func (s *Server) provisioner(ctx context.Context, m Control) {
|
||||
if s.standings == nil {
|
||||
// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
|
||||
// because handing it back would not give it anywhere to go.
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
st, err := ReadStanding(m.Subject(), m.Body())
|
||||
if err != nil {
|
||||
s.log.Printf("%v; ignored", err)
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
cleared, err := s.standings.Stood(ctx, st)
|
||||
what := fmt.Sprintf("%s's standing for %s", st.Module, st.Consumer)
|
||||
switch s.decide(ctx, m, what, "", "", err) {
|
||||
case Hold:
|
||||
return
|
||||
case Stale, GiveUp:
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.log.Printf("%s could not be kept: %v", what, err)
|
||||
} else if st.Failing {
|
||||
s.log.Printf("%s on %s is FAILING %s on %s (%s, %d attempts since %s): %s", st.Module,
|
||||
st.ProviderNode, st.Consumer, st.Node, st.Class, st.Attempts, st.Since.Format(time.RFC3339), st.Error)
|
||||
} else if cleared {
|
||||
s.log.Printf("%s on %s recovered %s", st.Module, st.ProviderNode, st.Consumer)
|
||||
}
|
||||
_ = m.Took()
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A provider's standing (novox/hq ADR 0224): who said it is read from the subject the bus let it
|
||||
// publish on, a failing one is kept and a recovery cleared, and a recovery is never lost to a store
|
||||
// that is away — said once, it would leave status naming a consumer that is fine.
|
||||
|
||||
type keptStandings struct {
|
||||
kept []Standing
|
||||
err error
|
||||
}
|
||||
|
||||
func (k *keptStandings) Stood(_ context.Context, s Standing) (bool, error) {
|
||||
if k.err != nil {
|
||||
return false, k.err
|
||||
}
|
||||
k.kept = append(k.kept, s)
|
||||
return !s.Failing, nil
|
||||
}
|
||||
|
||||
func standingSays(t *testing.T, in *fakeInbound, to *settled, subject string, body map[string]any) Control {
|
||||
t.Helper()
|
||||
m := in.sends(t, to, KindProvisioner, body).(*fakeControl)
|
||||
m.subject = subject
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T) {
|
||||
for subject, want := range map[string]string{
|
||||
"mesh.mod.keycloak.event.provisioner.failing": "keycloak",
|
||||
"mesh.mod.postgres.event.provisioner.recovered": "postgres",
|
||||
"mesh.mod.*.event.provisioner.failing": "*",
|
||||
} {
|
||||
got, ok := ProvisionerEmitter(subject)
|
||||
if !ok || got != want {
|
||||
t.Errorf("%s: %q %v", subject, got, ok)
|
||||
}
|
||||
if kind, _ := kindOfSubject(subject); kind != KindProvisioner {
|
||||
t.Errorf("%s decodes to %q", subject, kind)
|
||||
}
|
||||
}
|
||||
for _, subject := range []string{
|
||||
"mesh.mod.keycloak.event.provisioner.other",
|
||||
"mesh.mod.keycloak.event.client.created",
|
||||
"mesh.mod.a.b.event.provisioner.failing",
|
||||
"mesh.seat.keycloak.event.provisioner.failing",
|
||||
} {
|
||||
if _, ok := ProvisionerEmitter(subject); ok {
|
||||
t.Errorf("%s read as a provider's standing", subject)
|
||||
}
|
||||
}
|
||||
var follows int
|
||||
for _, s := range broker.ControllerFollows {
|
||||
if kind, _ := kindOfSubject(s); kind == KindProvisioner {
|
||||
follows++
|
||||
}
|
||||
}
|
||||
if follows != 2 {
|
||||
t.Fatalf("the controller follows %d standing subjects, want failing and recovered", follows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailingStandingIsKeptNamingTheEmitterFromTheSubject(t *testing.T) {
|
||||
s, in := serving()
|
||||
kept := &keptStandings{}
|
||||
if err := s.Watches(kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
to := &settled{}
|
||||
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
|
||||
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{
|
||||
"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
|
||||
"node": "home-server", "class": "credentials-rejected", "error": "401 invalid_grant",
|
||||
"since": since, "attempts": 31000,
|
||||
// A body naming another module is not believed: the subject is the bus's word.
|
||||
"module": "postgres",
|
||||
}))
|
||||
if !to.acked || len(kept.kept) != 1 {
|
||||
t.Fatalf("settled %+v, kept %+v", to, kept.kept)
|
||||
}
|
||||
got := kept.kept[0]
|
||||
if got.Module != "keycloak" || !got.Failing || got.Consumer != "mesh_home_grafana" || got.Node != "home-server" ||
|
||||
got.ProviderNode != "anchor" || got.Class != "credentials-rejected" || got.Attempts != 31000 || !got.Since.Equal(since) {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
|
||||
to = &settled{}
|
||||
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered", map[string]any{
|
||||
"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
|
||||
}))
|
||||
if !to.acked || len(kept.kept) != 2 || kept.kept[1].Failing {
|
||||
t.Fatalf("settled %+v, kept %+v", to, kept.kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecoveryIsHeldWhileTheStoreIsAway(t *testing.T) {
|
||||
s, in := serving()
|
||||
kept := &keptStandings{err: restarting}
|
||||
if err := s.Watches(kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
to := &settled{}
|
||||
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered",
|
||||
map[string]any{"consumer": "mesh_home_grafana"}))
|
||||
if !to.unsettled() || len(in.held) != 1 {
|
||||
t.Fatalf("a recovery was settled while the store was away: %+v", to)
|
||||
}
|
||||
kept.err = nil
|
||||
in.retries(t.Context(), s)
|
||||
if !to.acked || len(kept.kept) != 1 {
|
||||
t.Fatalf("the held recovery was not kept when the store came back: %+v %+v", to, kept.kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStandingThatNamesNoConsumerIsTakenAndForgotten(t *testing.T) {
|
||||
s, in := serving()
|
||||
kept := &keptStandings{}
|
||||
if err := s.Watches(kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
to := &settled{}
|
||||
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{}))
|
||||
if !to.acked || len(kept.kept) != 0 {
|
||||
t.Fatalf("%+v %+v", to, kept.kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) {
|
||||
s, in := serving()
|
||||
to := &settled{}
|
||||
s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing",
|
||||
map[string]any{"consumer": "x"}))
|
||||
if !to.acked {
|
||||
t.Fatal("a standing nothing keeps was left for the bus to hand over again")
|
||||
}
|
||||
}
|
||||
|
||||
// Over a real bus: a provider's standing published under its own module's namespace reaches the
|
||||
// controller through the events consumer's filter — the one wildcard filter on it — names the emitter
|
||||
// from the subject, and is acknowledged.
|
||||
func TestNatsAProvidersStandingReachesTheController(t *testing.T) {
|
||||
js := aBus(t)
|
||||
kept := &lockedStandings{}
|
||||
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
|
||||
if err := s.Follows(&toldAbout{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Watches(kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go func() { _ = s.Serve(ctx) }()
|
||||
eventually(t, "the controller's event consumer being made", func() bool {
|
||||
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
|
||||
return err == nil
|
||||
})
|
||||
|
||||
for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} {
|
||||
if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event,
|
||||
[]byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Somebody else's event under the same prefix is not the controller's to hear.
|
||||
if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "both standings being kept, in order, naming the emitter", func() bool {
|
||||
got := kept.all()
|
||||
return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing
|
||||
})
|
||||
eventually(t, "both being acknowledged and nothing else delivered", func() bool {
|
||||
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
|
||||
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
|
||||
})
|
||||
}
|
||||
|
||||
type lockedStandings struct {
|
||||
mu sync.Mutex
|
||||
kept []Standing
|
||||
}
|
||||
|
||||
func (l *lockedStandings) Stood(_ context.Context, s Standing) (bool, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.kept = append(l.kept, s)
|
||||
return !s.Failing, nil
|
||||
}
|
||||
|
||||
func (l *lockedStandings) all() []Standing {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return append([]Standing(nil), l.kept...)
|
||||
}
|
||||
@@ -32,8 +32,8 @@ const Requirement = "private-network"
|
||||
// Name is the module that answers it with WireGuard.
|
||||
//
|
||||
// **It writes no names.** A machine's mesh names are answered by the mesh's one resolver (novox/hq
|
||||
// ADR 0194), and /etc/hosts is the file of one module, the holder of `node-hosts-file` (ADR 0199): the
|
||||
// controller writes into no file another seat's holder owns. If the mesh ever needs a line there, it
|
||||
// ADR 0194), and /etc/hosts is the file of one module, the holder of `node-hostname` (ADR 0199,
|
||||
// ADR 0223): the controller writes into no file another seat's holder owns. If the mesh ever needs a line there, it
|
||||
// asks that holder to register it. This module asked for a `node-names` fact written into /etc/hosts
|
||||
// until 2026-10-05; the host gives that region back at the first push without it.
|
||||
const Name = "mesh-wireguard"
|
||||
|
||||
Reference in New Issue
Block a user