diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 11555243..3ca785dd 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -153,11 +153,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi standing[c.Key] = c } } - var hold *holding - if inv != nil { - if hold, err = readHolding(ctx, inv, open); err != nil { - return err - } + hold, err := readHoldingFor(ctx, inv, open) + if err != nil { + return err } var problems []string modules := make([]string, 0, len(unhealthy)) @@ -176,6 +174,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi // the operator's, never urgent, its words naming the act. if waits, waiting := operatorWait(m, unhealthy[m]); waiting { o := needsOperatorObservation(m, node, waits, unhealthy[m]) + // **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a + // login does: its consumers are held under it. + if hold != nil { + sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) + } seen[o.Key()] = true became[m] = kindNeedsOperator if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen { @@ -186,6 +189,20 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } continue } + // **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked + // wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the + // operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator + // condition stands beside the other, on the same looks; what follows judges the rest without the wait. + if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 { + o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m])) + seen[o.Key()] = true + if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen { + if _, err := k.Observe(ctx, o); err != nil { + problems = append(problems, err.Error()) + } + } + unhealthy[m] = rest + } // **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the // machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind, // so the gate never reads it as a fault of the build that happened to be sent beside it. @@ -259,7 +276,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module) } if on, held := heldOn[key]; held { - why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on) + why = fmt.Sprintf("what %s finds on %s waits on %s, which is not healthy: held under its condition", module, node, on) } // **A condition that became the other kind** is not "working again" (issue 318 review): its clearing // line says what it became. @@ -298,36 +315,72 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi // sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest // statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks. func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error { - var raisedAt *conditions.Condition - for i, c := range hold.open { - if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) { - raisedAt = &hold.open[i] - } - } - if raisedAt == nil { - return nil - } var rs []inventory.ResourceHealth for _, r := range hold.healths[p.Node].Resources { - if r.Module == p.Module && r.State == link.StateUnhealthy { + if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) { rs = append(rs, r) } } if len(rs) == 0 { return nil } - o := moduleUnhealthyObservation(p.Module, p.Node, rs) - if said, waits := personWait(p.Module, p.Node, rs); waits { - o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs) + // The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq + // issue 405), else that for the rest of it, a wait beside it said on its own. + var o conditions.Observation + if waits, waiting := operatorWait(p.Module, rs); waiting { + o = needsOperatorObservation(p.Module, p.Node, waits, rs) + } else { + _, rest := besideAWait(p.Module, rs) + o = moduleUnhealthyObservation(p.Module, p.Node, rest) + if said, waits := personWait(p.Module, p.Node, rest); waits { + o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest) + } } - if o.Key() != raisedAt.Key { - return nil // its own statement says it next + raised := false + for _, c := range hold.open { + raised = raised || c.Key == o.Key() + } + if !raised { + return nil // not open yet, or open as another kind: its own statement says it next } sayWaitingOn(&o, hold.waitersOn(p)) _, err := k.Observe(ctx, o) return err } +// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked +// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the +// module only waits (operatorWait says that whole). Pure. +func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) { + if _, only := operatorWait(module, rs); only { + return nil, rs + } + var waits []inventory.Wait + var rest []inventory.ResourceHealth + for _, r := range rs { + if r.Module == module && r.State == link.StateWaiting { + waits = append(waits, r.Waits...) + continue + } + rest = append(rest, r) + } + if len(waits) == 0 { + return nil, rs + } + return waits, rest +} + +// waitingOf is a module's waiting resources: the evidence of its wait. +func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth { + var out []inventory.ResourceHealth + for _, r := range rs { + if r.Module == module && r.State == link.StateWaiting { + out = append(out, r) + } + } + return out +} + // reloginKey is a module's relogin-needed condition on a machine. func reloginKey(module, node string) string { return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded) @@ -591,7 +644,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine) case link.StateUnhealthy: if on, held := f.heldOn[module+"@"+machine]; held { - return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind, + return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is not healthy", r.Kind, r.Resource, machine, on) } return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r)) diff --git a/cmd/mesh-controller/provider_hold.go b/cmd/mesh-controller/provider_hold.go index ea975bde..c9bd8c59 100644 --- a/cmd/mesh-controller/provider_hold.go +++ b/cmd/mesh-controller/provider_hold.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "slices" "sort" "strings" @@ -53,6 +54,15 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil } +// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a +// judging the record it holds under (novox/hq issue 405). +var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) { + if inv == nil { + return nil, nil + } + return readHolding(ctx, inv, open) +} + // heldFinding says a resource's state is a finding of its declared check that names a provision: what // may be held. Down and restarting are liveness, the resource's own. func heldFinding(r inventory.ResourceHealth) bool { @@ -106,17 +116,20 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue return catalogue.Chosen{}, false } -// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest -// statement says a resource of it is unhealthy. +// unhealthy says a provider is not healthy on the record: its unhealthy or needs-operator condition is open, or its +// machine's newest statement says a resource of it is unhealthy or waiting. **A provider waiting for the operator +// holds its consumers too** (novox/hq issue 405): its secret or setting not given, what its consumers find is no more +// theirs than when it is broken, and ADR 0240 rule 5 says it once, at the provider — here under its needs-operator +// condition, which names the act. func (h *holding) unhealthy(p catalogue.Chosen) bool { - key := moduleUnhealthyKey(p.Module, p.Node) + keys := []string{moduleUnhealthyKey(p.Module, p.Node), needsOperatorKey(p.Module, p.Node)} for _, c := range h.open { - if c.Key == key { + if slices.Contains(keys, c.Key) { return true } } for _, r := range h.healths[p.Node].Resources { - if r.Module == p.Module && r.State == link.StateUnhealthy { + if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) { return true } } diff --git a/cmd/mesh-controller/waiting_provider_test.go b/cmd/mesh-controller/waiting_provider_test.go new file mode 100644 index 00000000..c1554e28 --- /dev/null +++ b/cmd/mesh-controller/waiting_provider_test.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq +// issue 405, found in the review of ADR 0283's controller change). +// +// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of +// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session +// began before it was, and its unit failed in that account's own service manager. + +// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's +// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). +func waitingDatabase() inventory.ResourceHealth { + return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service", + State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", + Waits: []inventory.Wait{{Part: "the server", Secret: "licence", What: "the licence key of the database"}}} +} + +// failingConsumer is a consumer whose check that needs the database fails. +func failingConsumer(module string) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, + State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} +} + +// holdingOf is one reading of the record without a store: the newest statements, the open conditions, and each +// consumer's provider already looked up, as providerFor memoises it. +func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding { + h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{}} + for k, p := range bound { + h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true} + } + return h +} + +var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"} + +// (1) A provider whose only part not healthy waits for the operator holds the findings of its consumers under its +// needs-operator condition, as ADR 0240 rule 5 holds them under an unhealthy one. +func TestAProviderWaitingForTheOperatorHoldsItsConsumers(t *testing.T) { + healths := map[string]inventory.NodeHealth{ + "anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}, + "laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}}, + } + bound := map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase} + hold := holdingOf(nil, healths, bound) + p, held := hold.heldUnder("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}) + if !held || p != theDatabase { + t.Fatalf("a consumer of a provider waiting for the operator is not held under it: %v %v", p, held) + } + if got := hold.waitersOn(theDatabase); len(got) != 1 || got[0] != "shop on laptop" { + t.Fatalf("the waiting provider lists %v as waiting on it; want shop on laptop", got) + } + // Its needs-operator condition open is enough, as its unhealthy one is. + open := []conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}} + healths["anchor"] = inventory.NodeHealth{Node: "anchor"} + if _, held := holdingOf(open, healths, bound).heldUnder("laptop", "shop", + []inventory.ResourceHealth{failingConsumer("shop")}); !held { + t.Fatal("a consumer is not held under its provider's open needs-operator condition") + } + // A provider healthy holds nothing: what the consumer finds is its own. + healthy := waitingDatabase() + healthy.State, healthy.Waits = link.StateHealthy, nil + healths["anchor"] = inventory.NodeHealth{Node: "anchor", Resources: []inventory.ResourceHealth{healthy}} + if _, held := holdingOf(nil, healths, bound).heldUnder("laptop", "shop", + []inventory.ResourceHealth{failingConsumer("shop")}); held { + t.Fatal("a consumer of a healthy provider is held") + } +} + +// (1) The consumer raises nothing of its own, and the provider's needs-operator condition says who waits on it. +func TestAWaitingProvidersConditionListsWhoWaitsOnIt(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + healths := map[string]inventory.NodeHealth{ + "anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}, + "laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}}, + } + bound := map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase} + was := readHoldingFor + t.Cleanup(func() { readHoldingFor = was }) + readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { + return holdingOf(open, healths, bound), nil + } + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": {waitingDatabase()}}, + map[string]int{"db": look}, time.Now()); err != nil { + t.Fatal(err) + } + if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}, + map[string]int{"shop": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + open, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + var keys []string + var provider *conditions.Condition + for i, c := range open { + keys = append(keys, c.Key) + if c.Key == needsOperatorKey("db", "anchor") { + provider = &open[i] + } + } + if len(open) != 1 || provider == nil { + t.Fatalf("a provider waiting for the operator and a consumer failing on it raised %v; want the provider's "+ + "needs-operator alone", keys) + } + if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") { + t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said) + } + // Its words still name the act: it is the operator's, whoever waits on it. + if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") { + t.Fatalf("the provider's condition: %+v", provider) + } +} + +// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its +// wait for the password. +func reloginBesideAWait() []inventory.ResourceHealth { + return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"), + waitingResource(passwordWait)} +} + +// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the +// operator owes it. +func TestAWaitBesideAReloginIsSaid(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()}, + map[string]int{"mounts": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open)) + } + if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning { + t.Fatalf("the needs-operator beside the relogin: %+v", got) + } + relogged := false + for _, c := range open { + relogged = relogged || c.Key == reloginKey("mounts", "workstation") + } + if !relogged { + t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open)) + } + // The login done, the wait stays said and the relogin clears. + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}, + map[string]int{"mounts": 3}, time.Now()); err != nil { + t.Fatal(err) + } + got, open = needsOperatorOpen(t, k) + if got == nil || len(open) != 1 { + t.Fatalf("after the new login: %v", openKeysOf(open)) + } +} + +// (2) The same beside a directory used as found. +func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + found := foundDirectory("mounts", time.Now().Add(-time.Hour)) + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ + "mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open)) + } + asFound := false + for _, c := range open { + asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation") + } + if !asFound { + t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open)) + } +} + +// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among +// what fails. +func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process", + Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"} + for look := 1; look <= 2; look++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ + "mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open)) + } + var fault *conditions.Condition + for i, c := range open { + if c.Key == moduleUnhealthyKey("mounts", "workstation") { + fault = &open[i] + } + } + if fault == nil { + t.Fatalf("the fault is not said: %v", openKeysOf(open)) + } + if strings.Contains(fault.Summary, "mounts.watch") { + t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary) + } +} + +func openKeysOf(cs []conditions.Condition) []string { + var out []string + for _, c := range cs { + out = append(out, c.Key) + } + return out +}