Merge pull request 'A provider waiting for the operator holds its consumers, and a wait beside another condition is said (issue 405)' (#216) from fix/405-a-waiting-provider-holds-its-consumers into main
This commit was merged in pull request #216.
This commit is contained in:
@@ -121,7 +121,7 @@ type gateFacts struct {
|
||||
health map[string]inventory.NodeHealth
|
||||
healthErr error
|
||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||
heldOn map[string]string
|
||||
heldOn map[string]heldReading
|
||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||
groupsAdded map[string]bool
|
||||
@@ -137,6 +137,29 @@ type gateFacts struct {
|
||||
commits map[string]string
|
||||
}
|
||||
|
||||
// heldReading is the provider a module's findings are held under, as "<module> on <machine>", and, when that
|
||||
// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue
|
||||
// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283
|
||||
// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here.
|
||||
type heldReading struct {
|
||||
on, waits string
|
||||
}
|
||||
|
||||
// heldReadings is, per "<module>@<machine>" in every machine's newest statement, what its findings are held under.
|
||||
func heldReadings(hold *holding) map[string]heldReading {
|
||||
out := map[string]heldReading{}
|
||||
for machine := range hold.healths {
|
||||
for module, by := range hold.heldModules(machine) {
|
||||
reading := heldReading{on: by.provider.Module + " on " + by.provider.Node}
|
||||
if len(by.waits) > 0 {
|
||||
reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits)
|
||||
}
|
||||
out[module+"@"+machine] = reading
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that
|
||||
// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the
|
||||
// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer
|
||||
@@ -202,12 +225,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
|
||||
// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
|
||||
if f.healthErr == nil && f.openErr == nil {
|
||||
if hold, err := readHolding(ctx, inv, f.open); err == nil {
|
||||
f.heldOn = map[string]string{}
|
||||
for machine := range f.health {
|
||||
for module, p := range hold.heldModules(machine) {
|
||||
f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
|
||||
}
|
||||
}
|
||||
f.heldOn = heldReadings(hold)
|
||||
}
|
||||
}
|
||||
if theLease != nil {
|
||||
|
||||
@@ -153,11 +153,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
standing[c.Key] = c
|
||||
}
|
||||
}
|
||||
var hold *holding
|
||||
if inv != nil {
|
||||
if hold, err = readHolding(ctx, inv, open); err != nil {
|
||||
return err
|
||||
}
|
||||
hold, err := readHoldingFor(ctx, inv, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var problems []string
|
||||
modules := make([]string, 0, len(unhealthy))
|
||||
@@ -176,6 +174,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
// the operator's, never urgent, its words naming the act.
|
||||
if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
|
||||
o := needsOperatorObservation(m, node, waits, unhealthy[m])
|
||||
// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
|
||||
// login does: its consumers are held under it.
|
||||
if hold != nil {
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindNeedsOperator
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
@@ -186,6 +189,20 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
|
||||
// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
|
||||
// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
|
||||
// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
|
||||
if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
|
||||
o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
|
||||
seen[o.Key()] = true
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
|
||||
if _, err := k.Observe(ctx, o); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
unhealthy[m] = rest
|
||||
}
|
||||
// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
|
||||
// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
|
||||
// so the gate never reads it as a fault of the build that happened to be sent beside it.
|
||||
@@ -222,13 +239,24 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
o := moduleUnhealthyObservation(m, node, unhealthy[m])
|
||||
if hold != nil {
|
||||
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits.
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node
|
||||
if by, held := hold.heldWith(node, m, unhealthy[m]); held {
|
||||
// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
|
||||
// clearing line says which the provider is: unhealthy, or waiting for the operator (issue 405).
|
||||
p := by.provider
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which is unhealthy"
|
||||
if len(by.waits) > 0 {
|
||||
heldOn[o.Key()] = p.Module + " on " + p.Node + ", which waits for you"
|
||||
}
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
// A provider that waits for the operator for a part this finding cannot be matched to does not hold it
|
||||
// (novox/hq issue 405): raised as its own, and saying the provider waits, so neither is hidden.
|
||||
if p, waiting := hold.waitingUncovered(node, m, unhealthy[m]); waiting {
|
||||
o.Said += fmt.Sprintf("; %s on %s waits for you, but not for anything %s is known to need, so %s's "+
|
||||
"fault is said on its own", p.Module, p.Node, m, m)
|
||||
}
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindModuleUnhealthy
|
||||
@@ -259,7 +287,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
|
||||
}
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s: held under its condition", module, node, on)
|
||||
}
|
||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||
// line says what it became.
|
||||
@@ -298,36 +326,72 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
|
||||
// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
if raisedAt == nil {
|
||||
return nil
|
||||
}
|
||||
var rs []inventory.ResourceHealth
|
||||
for _, r := range hold.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
|
||||
rs = append(rs, r)
|
||||
}
|
||||
}
|
||||
if len(rs) == 0 {
|
||||
return nil
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||
// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
|
||||
// issue 405), else that for the rest of it, a wait beside it said on its own.
|
||||
var o conditions.Observation
|
||||
if waits, waiting := operatorWait(p.Module, rs); waiting {
|
||||
o = needsOperatorObservation(p.Module, p.Node, waits, rs)
|
||||
} else {
|
||||
_, rest := besideAWait(p.Module, rs)
|
||||
o = moduleUnhealthyObservation(p.Module, p.Node, rest)
|
||||
if said, waits := personWait(p.Module, p.Node, rest); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
|
||||
}
|
||||
}
|
||||
if o.Key() != raisedAt.Key {
|
||||
return nil // its own statement says it next
|
||||
raised := false
|
||||
for _, c := range hold.open {
|
||||
raised = raised || c.Key == o.Key()
|
||||
}
|
||||
if !raised {
|
||||
return nil // not open yet, or open as another kind: its own statement says it next
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(p))
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
|
||||
// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
|
||||
// module only waits (operatorWait says that whole). Pure.
|
||||
func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
|
||||
if _, only := operatorWait(module, rs); only {
|
||||
return nil, rs
|
||||
}
|
||||
var waits []inventory.Wait
|
||||
var rest []inventory.ResourceHealth
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.State == link.StateWaiting {
|
||||
waits = append(waits, r.Waits...)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, r)
|
||||
}
|
||||
if len(waits) == 0 {
|
||||
return nil, rs
|
||||
}
|
||||
return waits, rest
|
||||
}
|
||||
|
||||
// waitingOf is a module's waiting resources: the evidence of its wait.
|
||||
func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
|
||||
var out []inventory.ResourceHealth
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.State == link.StateWaiting {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reloginKey is a module's relogin-needed condition on a machine.
|
||||
func reloginKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||
@@ -399,12 +463,15 @@ func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
||||
}
|
||||
|
||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||
// waits on it, whether it is not working or waits for a new login.
|
||||
// waits on it, whether it is not working or waits for a new login. **A wait for the operator's secret or setting
|
||||
// stays a warning** (ADR 0283 decision 5, novox/hq issue 405): who waits on it is listed, and nothing escalates it.
|
||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
if len(waiters) == 0 {
|
||||
return
|
||||
}
|
||||
o.Severity = conditions.Urgent
|
||||
if o.Kind != kindNeedsOperator {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
@@ -562,7 +629,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
if waits && !f.groupsAdded[module] {
|
||||
waits = false
|
||||
}
|
||||
var found []string
|
||||
var found, onWaiting []string
|
||||
var forOperator []inventory.Wait
|
||||
for _, r := range resources {
|
||||
if r.Module != module {
|
||||
@@ -591,8 +658,16 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
|
||||
case link.StateUnhealthy:
|
||||
if on, held := f.heldOn[module+"@"+machine]; held {
|
||||
// **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a
|
||||
// person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the
|
||||
// operator's secret, whichever module owes it.
|
||||
if on.waits != "" {
|
||||
onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s",
|
||||
r.Kind, r.Resource, machine, on.on, on.waits))
|
||||
continue
|
||||
}
|
||||
return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
|
||||
r.Resource, machine, on)
|
||||
r.Resource, machine, on.on)
|
||||
}
|
||||
return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
|
||||
default:
|
||||
@@ -600,8 +675,8 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
reasonAfter(r.Reason))
|
||||
}
|
||||
}
|
||||
if waits || len(found) > 0 || len(forOperator) > 0 {
|
||||
var said []string
|
||||
if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 {
|
||||
said := onWaiting
|
||||
if waits {
|
||||
said = append(said, wait)
|
||||
}
|
||||
|
||||
@@ -53,6 +53,15 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition
|
||||
return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
|
||||
}
|
||||
|
||||
// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
|
||||
// judging the record it holds under (novox/hq issue 405).
|
||||
var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
if inv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return readHolding(ctx, inv, open)
|
||||
}
|
||||
|
||||
// heldFinding says a resource's state is a finding of its declared check that names a provision: what
|
||||
// may be held. Down and restarting are liveness, the resource's own.
|
||||
func heldFinding(r inventory.ResourceHealth) bool {
|
||||
@@ -106,42 +115,137 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
|
||||
// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
|
||||
// statement says a resource of it is unhealthy.
|
||||
func (h *holding) unhealthy(p catalogue.Chosen) bool {
|
||||
key := moduleUnhealthyKey(p.Module, p.Node)
|
||||
// providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its
|
||||
// machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement
|
||||
// says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then
|
||||
// unknown); else healthy.
|
||||
func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) {
|
||||
for _, c := range h.open {
|
||||
if c.Key == key {
|
||||
return true
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||
return true, false, nil
|
||||
}
|
||||
}
|
||||
for _, r := range h.healths[p.Node].Resources {
|
||||
if r.Module == p.Module && r.State == link.StateUnhealthy {
|
||||
return true
|
||||
if r.Module != p.Module {
|
||||
continue
|
||||
}
|
||||
switch r.State {
|
||||
case link.StateUnhealthy:
|
||||
return true, false, nil
|
||||
case link.StateWaiting:
|
||||
waiting = true
|
||||
waits = append(waits, r.Waits...)
|
||||
}
|
||||
}
|
||||
return false
|
||||
if !waiting {
|
||||
for _, c := range h.open {
|
||||
waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node)
|
||||
}
|
||||
}
|
||||
return false, waiting, waits
|
||||
}
|
||||
|
||||
// manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read.
|
||||
func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) {
|
||||
if h.shelf == nil && h.inv != nil {
|
||||
shelf, err := h.inv.Catalogue(h.ctx)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
h.shelf = shelf
|
||||
}
|
||||
m, ok := h.shelf[module]
|
||||
return m, ok
|
||||
}
|
||||
|
||||
// covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits
|
||||
// says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of
|
||||
// the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits
|
||||
// for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing
|
||||
// then is not held, since holding it would hide a fault that may be its own.
|
||||
func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait {
|
||||
credential := ""
|
||||
if m, ok := h.manifestOf(p.Module); ok {
|
||||
credential, _ = m.SharedCredentialOf(provision)
|
||||
}
|
||||
var out []inventory.Wait
|
||||
for _, w := range waits {
|
||||
if w.Part == provision || (w.Secret != "" && w.Secret == credential) {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the
|
||||
// operator, for the part the consumer needs — the waits that hold it.
|
||||
type heldUnderProvider struct {
|
||||
provider catalogue.Chosen
|
||||
// waits is set when every finding held waits on a provider that only waits for the operator: the consumer's
|
||||
// gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4).
|
||||
waits []inventory.Wait
|
||||
}
|
||||
|
||||
// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
|
||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
|
||||
// is the consumer's own.
|
||||
// of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the
|
||||
// operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own.
|
||||
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||
var on catalogue.Chosen
|
||||
by, held := h.heldWith(machine, module, rs)
|
||||
return by.provider, held
|
||||
}
|
||||
|
||||
func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) {
|
||||
var on heldUnderProvider
|
||||
onlyWaits := true
|
||||
for _, r := range rs {
|
||||
if r.State != link.StateUnhealthy {
|
||||
continue
|
||||
}
|
||||
if !heldFinding(r) {
|
||||
return catalogue.Chosen{}, false
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
p, ok := h.providerFor(machine, module, r.Needs)
|
||||
if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
|
||||
return catalogue.Chosen{}, false
|
||||
if !ok || (p.Node == machine && p.Module == module) {
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
on = p
|
||||
unhealthy, waiting, waits := h.providerState(p)
|
||||
switch {
|
||||
case unhealthy:
|
||||
onlyWaits = false
|
||||
case waiting:
|
||||
covered := h.covering(p, r.Needs, waits)
|
||||
if len(covered) == 0 {
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
on.waits = append(on.waits, covered...)
|
||||
default:
|
||||
return heldUnderProvider{}, false
|
||||
}
|
||||
on.provider = p
|
||||
}
|
||||
return on, on.Module != ""
|
||||
if !onlyWaits {
|
||||
on.waits = nil
|
||||
}
|
||||
return on, on.provider.Module != ""
|
||||
}
|
||||
|
||||
// waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the
|
||||
// finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says
|
||||
// the provider waits, so neither is hidden.
|
||||
func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
|
||||
for _, r := range rs {
|
||||
if r.State != link.StateUnhealthy || !heldFinding(r) {
|
||||
continue
|
||||
}
|
||||
p, ok := h.providerFor(machine, module, r.Needs)
|
||||
if !ok || (p.Node == machine && p.Module == module) {
|
||||
continue
|
||||
}
|
||||
if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return catalogue.Chosen{}, false
|
||||
}
|
||||
|
||||
// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
|
||||
@@ -165,8 +269,8 @@ func (h *holding) waitersOn(p catalogue.Chosen) []string {
|
||||
}
|
||||
|
||||
// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
|
||||
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||
out := map[string]catalogue.Chosen{}
|
||||
func (h *holding) heldModules(machine string) map[string]heldUnderProvider {
|
||||
out := map[string]heldUnderProvider{}
|
||||
byModule := map[string][]inventory.ResourceHealth{}
|
||||
for _, r := range h.healths[machine].Resources {
|
||||
if r.Module != "" && r.State == link.StateUnhealthy {
|
||||
@@ -174,7 +278,7 @@ func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
|
||||
}
|
||||
}
|
||||
for module, rs := range byModule {
|
||||
if on, held := h.heldUnder(machine, module, rs); held {
|
||||
if on, held := h.heldWith(machine, module, rs); held {
|
||||
out[module] = on
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
|
||||
// issue 405, found in the review of ADR 0283's controller change).
|
||||
//
|
||||
// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
|
||||
// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
|
||||
// began before it was, and its unit failed in that account's own service manager.
|
||||
|
||||
// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
|
||||
// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
|
||||
// names the part that waits by the provision it gives.
|
||||
func waitingDatabase() inventory.ResourceHealth {
|
||||
return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
|
||||
What: "the licence key of the database"})
|
||||
}
|
||||
|
||||
func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
|
||||
State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
|
||||
}
|
||||
|
||||
// backupsWait is a wait of the same provider for another part than the one its consumers need.
|
||||
var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
|
||||
|
||||
// failingConsumer is a consumer whose check that needs the database fails.
|
||||
func failingConsumer(module string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
}
|
||||
|
||||
// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
|
||||
// catalogue, and each consumer's provider already looked up, as providerFor memoises it.
|
||||
func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
|
||||
h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
|
||||
shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}}}
|
||||
for k, p := range bound {
|
||||
h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
|
||||
|
||||
var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
|
||||
|
||||
func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
|
||||
return map[string]inventory.NodeHealth{
|
||||
"anchor": {Node: "anchor", Resources: provider},
|
||||
"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
|
||||
}
|
||||
}
|
||||
|
||||
// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
|
||||
// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
|
||||
// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
|
||||
func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
|
||||
shop := []inventory.ResourceHealth{failingConsumer("shop")}
|
||||
unhealthyDB := waitingDatabase()
|
||||
unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
||||
healthyDB := waitingDatabase()
|
||||
healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
|
||||
byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
|
||||
Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
|
||||
byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{
|
||||
Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
hold *holding
|
||||
held bool
|
||||
onlyWaits bool
|
||||
uncovered bool
|
||||
}{
|
||||
{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
|
||||
{"the wait is for the provision's shared credential", byCredential, true, true, false},
|
||||
{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
|
||||
{"only the needs-operator condition, its parts not said", holdingOf(
|
||||
[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
|
||||
shopFailingBeside(), shopOnTheDatabase), false, false, true},
|
||||
{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
|
||||
{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
|
||||
} {
|
||||
by, held := c.hold.heldWith("laptop", "shop", shop)
|
||||
if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
|
||||
t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
|
||||
by.waits, c.held, c.onlyWaits)
|
||||
}
|
||||
if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
|
||||
t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
|
||||
// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
|
||||
func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
provider inventory.ResourceHealth
|
||||
want health
|
||||
says []string
|
||||
}{
|
||||
{"a provider that only waits", waitingDatabase(), healthPerson,
|
||||
[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
|
||||
{"an unhealthy provider", func() inventory.ResourceHealth {
|
||||
r := waitingDatabase()
|
||||
r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
||||
return r
|
||||
}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
|
||||
} {
|
||||
healths := shopFailingBeside(c.provider)
|
||||
for m, h := range healths {
|
||||
h.HeardAt = now
|
||||
healths[m] = h
|
||||
}
|
||||
f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
|
||||
h, why := moduleHealthWord("shop", "laptop", since, f)
|
||||
if h != c.want {
|
||||
t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
|
||||
continue
|
||||
}
|
||||
for _, s := range c.says {
|
||||
if !strings.Contains(why, s) {
|
||||
t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
|
||||
// as the statements do.
|
||||
func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
|
||||
byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
was := readHoldingFor
|
||||
t.Cleanup(func() { readHoldingFor = was })
|
||||
healths := byProvider
|
||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
return holdingOf(open, healths, shopOnTheDatabase), nil
|
||||
}
|
||||
for look := 1; look <= 2; look++ {
|
||||
healths = byProvider
|
||||
if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
|
||||
map[string]int{"db": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for look := 1; look <= 2; look++ {
|
||||
healths = byConsumer
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
|
||||
map[string]int{"shop": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
|
||||
for i, c := range open {
|
||||
if c.Key == key {
|
||||
return &open[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
|
||||
// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
|
||||
// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
|
||||
func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
|
||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
|
||||
provider := conditionOf(open, needsOperatorKey("db", "anchor"))
|
||||
if len(open) != 1 || provider == nil {
|
||||
t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
|
||||
"provider's needs-operator alone", openKeysOf(open))
|
||||
}
|
||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||
t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
|
||||
}
|
||||
if provider.Severity != conditions.Warning {
|
||||
t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
|
||||
}
|
||||
if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
|
||||
t.Fatalf("the provider's condition: %+v", provider)
|
||||
}
|
||||
}
|
||||
|
||||
// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
|
||||
func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
backups := waitingDatabaseFor(backupsWait)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
|
||||
consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
|
||||
if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
|
||||
t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
|
||||
}
|
||||
if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") {
|
||||
t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
|
||||
// wait for the password.
|
||||
func reloginBesideAWait() []inventory.ResourceHealth {
|
||||
return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
|
||||
waitingResource(passwordWait)}
|
||||
}
|
||||
|
||||
// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
|
||||
// operator owes it.
|
||||
func TestAWaitBesideAReloginIsSaid(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
|
||||
map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
|
||||
}
|
||||
if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
|
||||
t.Fatalf("the needs-operator beside the relogin: %+v", got)
|
||||
}
|
||||
relogged := false
|
||||
for _, c := range open {
|
||||
relogged = relogged || c.Key == reloginKey("mounts", "workstation")
|
||||
}
|
||||
if !relogged {
|
||||
t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
|
||||
}
|
||||
// The login done, the wait stays said and the relogin clears.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
|
||||
map[string]int{"mounts": 3}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, open = needsOperatorOpen(t, k)
|
||||
if got == nil || len(open) != 1 {
|
||||
t.Fatalf("after the new login: %v", openKeysOf(open))
|
||||
}
|
||||
}
|
||||
|
||||
// (2) The same beside a directory used as found.
|
||||
func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
found := foundDirectory("mounts", time.Now().Add(-time.Hour))
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
||||
"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
|
||||
}
|
||||
asFound := false
|
||||
for _, c := range open {
|
||||
asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
|
||||
}
|
||||
if !asFound {
|
||||
t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
|
||||
}
|
||||
}
|
||||
|
||||
// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
|
||||
// what fails.
|
||||
func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
||||
Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
||||
"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, open := needsOperatorOpen(t, k)
|
||||
if got == nil {
|
||||
t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
|
||||
}
|
||||
var fault *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == moduleUnhealthyKey("mounts", "workstation") {
|
||||
fault = &open[i]
|
||||
}
|
||||
}
|
||||
if fault == nil {
|
||||
t.Fatalf("the fault is not said: %v", openKeysOf(open))
|
||||
}
|
||||
if strings.Contains(fault.Summary, "mounts.watch") {
|
||||
t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func openKeysOf(cs []conditions.Condition) []string {
|
||||
var out []string
|
||||
for _, c := range cs {
|
||||
out = append(out, c.Key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying
|
||||
// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review).
|
||||
func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
start := time.Now().Add(-time.Second)
|
||||
healthy := waitingDatabase()
|
||||
healthy.State, healthy.Waits = link.StateHealthy, nil
|
||||
healths := shopFailingBeside(healthy)
|
||||
was := readHoldingFor
|
||||
t.Cleanup(func() { readHoldingFor = was })
|
||||
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
||||
return holdingOf(open, healths, shopOnTheDatabase), nil
|
||||
}
|
||||
shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}
|
||||
for look := 1; look <= 2; look++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil {
|
||||
t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open))
|
||||
}
|
||||
healths = shopFailingBeside(waitingDatabase())
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var why string
|
||||
for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); {
|
||||
events, err := k.HistorySince(ctx, start)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range events {
|
||||
if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared {
|
||||
why = e.Why
|
||||
}
|
||||
}
|
||||
if why == "" {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(why, "waits on db on anchor, which waits for you") {
|
||||
t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user