diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 235a53b7..36eefc4d 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -121,7 +121,7 @@ type gateFacts struct { health map[string]inventory.NodeHealth healthErr error // heldOn is, per "@", the provider its findings are held under (ADR 0240 rule 5). - heldOn map[string]string + heldOn map[string]heldReading // groupsAdded is, per module, whether the move judged puts an account in a group its previous build did // not (issue 318 review): the only move whose wait for a new login is excused. groupsAdded map[string]bool @@ -137,6 +137,29 @@ type gateFacts struct { commits map[string]string } +// heldReading is the provider a module's findings are held under, as " on ", and, when that +// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue +// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283 +// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here. +type heldReading struct { + on, waits string +} + +// heldReadings is, per "@" in every machine's newest statement, what its findings are held under. +func heldReadings(hold *holding) map[string]heldReading { + out := map[string]heldReading{} + for machine := range hold.healths { + for module, by := range hold.heldModules(machine) { + reading := heldReading{on: by.provider.Module + " on " + by.provider.Node} + if len(by.waits) > 0 { + reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits) + } + out[module+"@"+machine] = reading + } + } + return out +} + // reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that // declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the // declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer @@ -202,12 +225,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string) // Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail. if f.healthErr == nil && f.openErr == nil { if hold, err := readHolding(ctx, inv, f.open); err == nil { - f.heldOn = map[string]string{} - for machine := range f.health { - for module, p := range hold.heldModules(machine) { - f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node - } - } + f.heldOn = heldReadings(hold) } } if theLease != nil { diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 11555243..2a65f06a 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -153,11 +153,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi standing[c.Key] = c } } - var hold *holding - if inv != nil { - if hold, err = readHolding(ctx, inv, open); err != nil { - return err - } + hold, err := readHoldingFor(ctx, inv, open) + if err != nil { + return err } var problems []string modules := make([]string, 0, len(unhealthy)) @@ -176,6 +174,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi // the operator's, never urgent, its words naming the act. if waits, waiting := operatorWait(m, unhealthy[m]); waiting { o := needsOperatorObservation(m, node, waits, unhealthy[m]) + // **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a + // login does: its consumers are held under it. + if hold != nil { + sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) + } seen[o.Key()] = true became[m] = kindNeedsOperator if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen { @@ -186,6 +189,20 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } continue } + // **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked + // wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the + // operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator + // condition stands beside the other, on the same looks; what follows judges the rest without the wait. + if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 { + o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m])) + seen[o.Key()] = true + if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen { + if _, err := k.Observe(ctx, o); err != nil { + problems = append(problems, err.Error()) + } + } + unhealthy[m] = rest + } // **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the // machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind, // so the gate never reads it as a fault of the build that happened to be sent beside it. @@ -222,13 +239,24 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } o := moduleUnhealthyObservation(m, node, unhealthy[m]) if hold != nil { - if p, held := hold.heldUnder(node, m, unhealthy[m]); held { - // Held under the provider's condition: nothing of its own, and the provider's says it waits. - heldOn[o.Key()] = p.Module + " on " + p.Node + if by, held := hold.heldWith(node, m, unhealthy[m]); held { + // Held under the provider's condition: nothing of its own, and the provider's says it waits. The + // clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405). + p := by.provider + heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy" + if len(by.waits) > 0 { + heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you" + } providers[p] = true continue } sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) + // A provider that waits for the operator for a part this finding cannot be matched to does not hold it + // (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden. + if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting { + o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+ + "fault is said on its own", p.Module, p.Node, m, m) + } } seen[o.Key()] = true became[m] = kindModuleUnhealthy @@ -259,7 +287,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module) } if on, held := heldOn[key]; held { - why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on) + why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on) } // **A condition that became the other kind** is not "working again" (issue 318 review): its clearing // line says what it became. @@ -298,36 +326,72 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi // sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest // statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks. func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error { - var raisedAt *conditions.Condition - for i, c := range hold.open { - if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) { - raisedAt = &hold.open[i] - } - } - if raisedAt == nil { - return nil - } var rs []inventory.ResourceHealth for _, r := range hold.healths[p.Node].Resources { - if r.Module == p.Module && r.State == link.StateUnhealthy { + if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) { rs = append(rs, r) } } if len(rs) == 0 { return nil } - o := moduleUnhealthyObservation(p.Module, p.Node, rs) - if said, waits := personWait(p.Module, p.Node, rs); waits { - o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs) + // The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq + // issue 405), else that for the rest of it, a wait beside it said on its own. + var o conditions.Observation + if waits, waiting := operatorWait(p.Module, rs); waiting { + o = needsOperatorObservation(p.Module, p.Node, waits, rs) + } else { + _, rest := besideAWait(p.Module, rs) + o = moduleUnhealthyObservation(p.Module, p.Node, rest) + if said, waits := personWait(p.Module, p.Node, rest); waits { + o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest) + } } - if o.Key() != raisedAt.Key { - return nil // its own statement says it next + raised := false + for _, c := range hold.open { + raised = raised || c.Key == o.Key() + } + if !raised { + return nil // not open yet, or open as another kind: its own statement says it next } sayWaitingOn(&o, hold.waitersOn(p)) _, err := k.Observe(ctx, o) return err } +// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked +// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the +// module only waits (operatorWait says that whole). Pure. +func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) { + if _, only := operatorWait(module, rs); only { + return nil, rs + } + var waits []inventory.Wait + var rest []inventory.ResourceHealth + for _, r := range rs { + if r.Module == module && r.State == link.StateWaiting { + waits = append(waits, r.Waits...) + continue + } + rest = append(rest, r) + } + if len(waits) == 0 { + return nil, rs + } + return waits, rest +} + +// waitingOf is a module's waiting resources: the evidence of its wait. +func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth { + var out []inventory.ResourceHealth + for _, r := range rs { + if r.Module == module && r.State == link.StateWaiting { + out = append(out, r) + } + } + return out +} + // reloginKey is a module's relogin-needed condition on a machine. func reloginKey(module, node string) string { return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded) @@ -399,12 +463,15 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string { } // sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone -// waits on it, whether it is not working or waits for a new login. +// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting +// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it. func sayWaitingOn(o *conditions.Observation, waiters []string) { if len(waiters) == 0 { return } - o.Severity = conditions.Urgent + if o.Kind != kindNeedsOperator { + o.Severity = conditions.Urgent + } o.Said += "; " + waitingWords(waiters) o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters)) @@ -562,7 +629,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea if waits && !f.groupsAdded[module] { waits = false } - var found []string + var found, onWaiting []string var forOperator []inventory.Wait for _, r := range resources { if r.Module != module { @@ -591,8 +658,16 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine) case link.StateUnhealthy: if on, held := f.heldOn[module+"@"+machine]; held { + // **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a + // person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the + // operator's secret, whichever module owes it. + if on.waits != "" { + onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s", + r.Kind, r.Resource, machine, on.on, on.waits)) + continue + } return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind, - r.Resource, machine, on) + r.Resource, machine, on.on) } return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r)) default: @@ -600,8 +675,8 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea reasonAfter(r.Reason)) } } - if waits || len(found) > 0 || len(forOperator) > 0 { - var said []string + if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 { + said := onWaiting if waits { said = append(said, wait) } diff --git a/cmd/mesh-controller/provider_hold.go b/cmd/mesh-controller/provider_hold.go index ea975bde..51651015 100644 --- a/cmd/mesh-controller/provider_hold.go +++ b/cmd/mesh-controller/provider_hold.go @@ -53,6 +53,15 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil } +// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a +// judging the record it holds under (novox/hq issue 405). +var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) { + if inv == nil { + return nil, nil + } + return readHolding(ctx, inv, open) +} + // heldFinding says a resource's state is a finding of its declared check that names a provision: what // may be held. Down and restarting are liveness, the resource's own. func heldFinding(r inventory.ResourceHealth) bool { @@ -106,42 +115,137 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue return catalogue.Chosen{}, false } -// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest -// statement says a resource of it is unhealthy. -func (h *holding) unhealthy(p catalogue.Chosen) bool { - key := moduleUnhealthyKey(p.Module, p.Node) +// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its +// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement +// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then +// unknown); else healthy. +func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) { for _, c := range h.open { - if c.Key == key { - return true + if c.Key == moduleUnhealthyKey(p.Module, p.Node) { + return true, false, nil } } for _, r := range h.healths[p.Node].Resources { - if r.Module == p.Module && r.State == link.StateUnhealthy { - return true + if r.Module != p.Module { + continue + } + switch r.State { + case link.StateUnhealthy: + return true, false, nil + case link.StateWaiting: + waiting = true + waits = append(waits, r.Waits...) } } - return false + if !waiting { + for _, c := range h.open { + waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node) + } + } + return false, waiting, waits +} + +// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read. +func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) { + if h.shelf == nil && h.inv != nil { + shelf, err := h.inv.Catalogue(h.ctx) + if err != nil { + return catalogue.Manifest{}, false + } + h.shelf = shelf + } + m, ok := h.shelf[module] + return m, ok +} + +// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits +// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of +// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits +// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing +// then is not held, since holding it would hide a fault that may be its own. +func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait { + credential := "" + if m, ok := h.manifestOf(p.Module); ok { + credential, _ = m.SharedCredentialOf(provision) + } + var out []inventory.Wait + for _, w := range waits { + if w.Part == provision || (w.Secret != "" && w.Secret == credential) { + out = append(out, w) + } + } + return out +} + +// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the +// operator, for the part the consumer needs — the waits that hold it. +type heldUnderProvider struct { + provider catalogue.Chosen + // waits is set when every finding held waits on a provider that only waits for the operator: the consumer's + // gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4). + waits []inventory.Wait } // heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding -// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any -// is the consumer's own. +// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the +// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own. func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) { - var on catalogue.Chosen + by, held := h.heldWith(machine, module, rs) + return by.provider, held +} + +func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) { + var on heldUnderProvider + onlyWaits := true for _, r := range rs { if r.State != link.StateUnhealthy { continue } if !heldFinding(r) { - return catalogue.Chosen{}, false + return heldUnderProvider{}, false } p, ok := h.providerFor(machine, module, r.Needs) - if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) { - return catalogue.Chosen{}, false + if !ok || (p.Node == machine && p.Module == module) { + return heldUnderProvider{}, false } - on = p + unhealthy, waiting, waits := h.providerState(p) + switch { + case unhealthy: + onlyWaits = false + case waiting: + covered := h.covering(p, r.Needs, waits) + if len(covered) == 0 { + return heldUnderProvider{}, false + } + on.waits = append(on.waits, covered...) + default: + return heldUnderProvider{}, false + } + on.provider = p } - return on, on.Module != "" + if !onlyWaits { + on.waits = nil + } + return on, on.provider.Module != "" +} + +// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the +// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says +// the provider waits, so neither is hidden. +func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) { + for _, r := range rs { + if r.State != link.StateUnhealthy || !heldFinding(r) { + continue + } + p, ok := h.providerFor(machine, module, r.Needs) + if !ok || (p.Node == machine && p.Module == module) { + continue + } + if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 { + return p, true + } + } + return catalogue.Chosen{}, false } // waitersOn is every consumer held under a provider, as " on ", sorted. @@ -165,8 +269,8 @@ func (h *holding) waitersOn(p catalogue.Chosen) []string { } // heldModules is, for one machine's statement, each module whose finding is held, with the provider. -func (h *holding) heldModules(machine string) map[string]catalogue.Chosen { - out := map[string]catalogue.Chosen{} +func (h *holding) heldModules(machine string) map[string]heldUnderProvider { + out := map[string]heldUnderProvider{} byModule := map[string][]inventory.ResourceHealth{} for _, r := range h.healths[machine].Resources { if r.Module != "" && r.State == link.StateUnhealthy { @@ -174,7 +278,7 @@ func (h *holding) heldModules(machine string) map[string]catalogue.Chosen { } } for module, rs := range byModule { - if on, held := h.heldUnder(machine, module, rs); held { + if on, held := h.heldWith(machine, module, rs); held { out[module] = on } } diff --git a/cmd/mesh-controller/waiting_provider_test.go b/cmd/mesh-controller/waiting_provider_test.go new file mode 100644 index 00000000..7ab5e780 --- /dev/null +++ b/cmd/mesh-controller/waiting_provider_test.go @@ -0,0 +1,375 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq +// issue 405, found in the review of ADR 0283's controller change). +// +// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of +// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session +// began before it was, and its unit failed in that account's own service manager. + +// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's +// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait +// names the part that waits by the provision it gives. +func waitingDatabase() inventory.ResourceHealth { + return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence", + What: "the licence key of the database"}) +} + +func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service", + State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits} +} + +// backupsWait is a wait of the same provider for another part than the one its consumers need. +var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"} + +// failingConsumer is a consumer whose check that needs the database fails. +func failingConsumer(module string) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, + State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} +} + +// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the +// catalogue, and each consumer's provider already looked up, as providerFor memoises it. +func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding { + h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{}, + shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", + Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}}} + for k, p := range bound { + h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true} + } + return h +} + +var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"} + +var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase} + +func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth { + return map[string]inventory.NodeHealth{ + "anchor": {Node: "anchor", Resources: provider}, + "laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}}, + } +} + +// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the +// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is +// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong). +func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) { + shop := []inventory.ResourceHealth{failingConsumer("shop")} + unhealthyDB := waitingDatabase() + unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused" + healthyDB := waitingDatabase() + healthyDB.State, healthyDB.Waits = link.StateHealthy, nil + byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server", + Secret: "licence", What: "the licence key"})), shopOnTheDatabase) + byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{ + Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}} + for _, c := range []struct { + name string + hold *holding + held bool + onlyWaits bool + uncovered bool + }{ + {"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false}, + {"the wait is for the provision's shared credential", byCredential, true, true, false}, + {"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true}, + {"only the needs-operator condition, its parts not said", holdingOf( + []conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}}, + shopFailingBeside(), shopOnTheDatabase), false, false, true}, + {"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false}, + {"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false}, + } { + by, held := c.hold.heldWith("laptop", "shop", shop) + if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits { + t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider, + by.waits, c.held, c.onlyWaits) + } + if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered { + t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered) + } + } +} + +// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a +// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before. +func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) { + now := time.Now() + since := now.Add(-time.Minute) + for _, c := range []struct { + name string + provider inventory.ResourceHealth + want health + says []string + }{ + {"a provider that only waits", waitingDatabase(), healthPerson, + []string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}}, + {"an unhealthy provider", func() inventory.ResourceHealth { + r := waitingDatabase() + r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused" + return r + }(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}}, + } { + healths := shopFailingBeside(c.provider) + for m, h := range healths { + h.HeardAt = now + healths[m] = h + } + f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))} + h, why := moduleHealthWord("shop", "laptop", since, f) + if h != c.want { + t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want) + continue + } + for _, s := range c.says { + if !strings.Contains(why, s) { + t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s) + } + } + } +} + +// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes +// as the statements do. +func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth, + byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition { + t.Helper() + ctx := t.Context() + was := readHoldingFor + t.Cleanup(func() { readHoldingFor = was }) + healths := byProvider + readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { + return holdingOf(open, healths, shopOnTheDatabase), nil + } + for look := 1; look <= 2; look++ { + healths = byProvider + if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider}, + map[string]int{"db": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + for look := 1; look <= 2; look++ { + healths = byConsumer + if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}, + map[string]int{"shop": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + open, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + return open +} + +func conditionOf(open []conditions.Condition, key string) *conditions.Condition { + for i, c := range open { + if c.Key == key { + return &open[i] + } + } + return nil +} + +// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and +// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's, +// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved. +func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) { + k, _ := withConditionsInMemory(t) + open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()}, + map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase())) + provider := conditionOf(open, needsOperatorKey("db", "anchor")) + if len(open) != 1 || provider == nil { + t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+ + "provider's needs-operator alone", openKeysOf(open)) + } + if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") { + t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said) + } + if provider.Severity != conditions.Warning { + t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity) + } + if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") { + t.Fatalf("the provider's condition: %+v", provider) + } +} + +// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits. +func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) { + k, _ := withConditionsInMemory(t) + backups := waitingDatabaseFor(backupsWait) + open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups)) + consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop")) + if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil { + t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open)) + } + if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") { + t.Fatalf("the consumer's condition does not say its provider waits: %s", said) + } +} + +// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its +// wait for the password. +func reloginBesideAWait() []inventory.ResourceHealth { + return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"), + waitingResource(passwordWait)} +} + +// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the +// operator owes it. +func TestAWaitBesideAReloginIsSaid(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()}, + map[string]int{"mounts": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open)) + } + if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning { + t.Fatalf("the needs-operator beside the relogin: %+v", got) + } + relogged := false + for _, c := range open { + relogged = relogged || c.Key == reloginKey("mounts", "workstation") + } + if !relogged { + t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open)) + } + // The login done, the wait stays said and the relogin clears. + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}, + map[string]int{"mounts": 3}, time.Now()); err != nil { + t.Fatal(err) + } + got, open = needsOperatorOpen(t, k) + if got == nil || len(open) != 1 { + t.Fatalf("after the new login: %v", openKeysOf(open)) + } +} + +// (2) The same beside a directory used as found. +func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + found := foundDirectory("mounts", time.Now().Add(-time.Hour)) + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ + "mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open)) + } + asFound := false + for _, c := range open { + asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation") + } + if !asFound { + t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open)) + } +} + +// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among +// what fails. +func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process", + Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"} + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ + "mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open)) + } + var fault *conditions.Condition + for i, c := range open { + if c.Key == moduleUnhealthyKey("mounts", "workstation") { + fault = &open[i] + } + } + if fault == nil { + t.Fatalf("the fault is not said: %v", openKeysOf(open)) + } + if strings.Contains(fault.Summary, "mounts.watch") { + t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary) + } +} + +func openKeysOf(cs []conditions.Condition) []string { + var out []string + for _, c := range cs { + out = append(out, c.Key) + } + return out +} + +// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying +// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review). +func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + start := time.Now().Add(-time.Second) + healthy := waitingDatabase() + healthy.State, healthy.Waits = link.StateHealthy, nil + healths := shopFailingBeside(healthy) + was := readHoldingFor + t.Cleanup(func() { readHoldingFor = was }) + readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { + return holdingOf(open, healths, shopOnTheDatabase), nil + } + shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}} + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil { + t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open)) + } + healths = shopFailingBeside(waitingDatabase()) + if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil { + t.Fatal(err) + } + var why string + for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); { + events, err := k.HistorySince(ctx, start) + if err != nil { + t.Fatal(err) + } + for _, e := range events { + if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared { + why = e.Why + } + } + if why == "" { + time.Sleep(10 * time.Millisecond) + } + } + if !strings.Contains(why, "waits on db on anchor, which waits for you") { + t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index ff5be865..fb9010bc 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -1076,7 +1076,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, // credential the mesh had replaced, because its manifest restarted it on its // environment file and nobody had thought to name the credential too. Composed here so // no manifest has to say it, for a container or a daemon that names the secret's path. - if reads := secretsReadBy(copied, m); len(reads) > 0 { + if reads := secretsReadBy(copied, m, with.Needed[m.Module]); len(reads) > 0 { copied["restart-on"] = withRestartOn(copied["restart-on"], reads) } // **A version prepares its state before it runs** (novox/hq ADR 0135). Derived from the @@ -2392,7 +2392,12 @@ func portOfEndpoint(values map[string]any, ports map[string]int) { // — named in its volumes, its environment or its env-files by the secret's placed path — as // restart-on ids. Nothing for other shapes, and nothing for a scheduled or run-once process, which // the host refuses a restart-on for (it runs again anyway, and reads the file afresh). -func secretsReadBy(resource map[string]any, m Manifest) []string { +// +// **A member of a secret family given here is read the same way** (novox/hq issue 405, ADR 0283 decision 6): +// it is an own secret placed at its own path, and a container that mounted it would keep the value it +// started with when the operator gives it again. given is the module's own secrets sealed to this +// machine; a member not given is no file, so nothing restarts on it. +func secretsReadBy(resource map[string]any, m Manifest, given map[string]string) []string { kind := fmt.Sprint(resource["type"]) if kind != "container" && kind != "process" { return nil @@ -2404,9 +2409,18 @@ func secretsReadBy(resource map[string]any, m Manifest) []string { for _, key := range []string{"volumes", "env", "env-file"} { mentioned = append(mentioned, stringsIn(resource[key])...) } + paths := map[string]string{} + for name, own := range m.OwnSecrets.Plain() { + paths[name] = own.Path + } + for name, sealed := range given { + if own, family, ok := m.OwnSecrets.Lookup(name); ok && family != "" && sealed != "" { + paths[name] = own.Path + } + } var out []string - for _, name := range sortedKeys(m.OwnSecrets.Plain()) { - path := m.OwnSecrets[name].Path + for _, name := range sortedKeys(paths) { + path := paths[name] if path == "" { continue } diff --git a/internal/catalogue/secret_restart_test.go b/internal/catalogue/secret_restart_test.go index 5460c096..560eb52d 100644 --- a/internal/catalogue/secret_restart_test.go +++ b/internal/catalogue/secret_restart_test.go @@ -50,3 +50,41 @@ func TestAContainerReadingAnOwnSecretIsRestartedWhenItChanges(t *testing.T) { t.Fatalf("a container that reads no secret was given one to restart on: %v", by["agent.other"]["restart-on"]) } } + +// A member of a secret family is an own secret too (novox/hq issue 405, ADR 0283 decision 6): a container that +// reads a member given is restarted when it changes, as for a secret declared by name. A member not given is no +// file, so nothing is restarted on it. +func TestAContainerReadingAFamilyMemberIsRestartedWhenItChanges(t *testing.T) { + m := Manifest{Module: "mounts", Version: "1", + OwnSecrets: OwnSecrets{"smb-password-*": {Path: "/var/lib/mesh/mounts/smb-password-*.secret", IssuedBy: IssuedOutside}}, + Resources: []map[string]any{ + {"id": "games", "type": "container", "name": "mounts-games", "network": "host", + "image": "registry.example/mounts@sha256:" + strings.Repeat("a", 64), + "volumes": []any{"/var/lib/mesh/mounts/smb-password-games.secret:/run/password:ro"}}, + {"id": "library", "type": "container", "name": "mounts-library", "network": "host", + "image": "registry.example/mounts@sha256:" + strings.Repeat("a", 64), + "volumes": []any{"/var/lib/mesh/mounts/smb-password-library.secret:/run/password:ro"}}, + }} + got, err := Resolve(shelf(m), []string{m.Module}, + Node{Name: "anchor", At: "10.0.0.1", Capabilities: map[string]bool{"container-runtime": true}}, World{}) + if err != nil { + t.Fatal(err) + } + out, err := got.Declaration(Rendering{Needed: map[string]map[string]string{"mounts": {"smb-password-games": "SEALED"}}}) + if err != nil { + t.Fatal(err) + } + by := map[string]map[string]any{} + for _, r := range out { + by[r["id"].(string)] = r + } + if want := []any{"mounts.needs-smb-password-games"}; !reflect.DeepEqual(by["mounts.games"]["restart-on"], want) { + t.Fatalf("a container reading a member given is not restarted on it: %v", by["mounts.games"]["restart-on"]) + } + if _, placed := by["mounts.needs-smb-password-games"]; !placed { + t.Fatalf("the member given is not placed, so a restart-on names nothing: %v", out) + } + if _, has := by["mounts.library"]["restart-on"]; has { + t.Fatalf("a container reading a member not given was given a restart-on naming nothing: %v", by["mounts.library"]["restart-on"]) + } +}