jschoubben
ed45cc6415
A provider whose wait fails the check raises unhealthy, but the holding read its stored statement with the wait unchecked: sayWaiters built the needs-operator key, found it not open and listed no held consumer. The holding now reads each statement as judged (ADR 0283 decision 3), in sayWaiters and in the provider's state its consumers are held by.
385 lines
17 KiB
Go
385 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
|
|
// issue 405, found in the review of ADR 0283's controller change).
|
|
//
|
|
// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
|
|
// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
|
|
// began before it was, and its unit failed in that account's own service manager.
|
|
|
|
// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
|
|
// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
|
|
// names the part that waits by the provision it gives.
|
|
func waitingDatabase() inventory.ResourceHealth {
|
|
return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
|
|
What: "the licence key of the database"})
|
|
}
|
|
|
|
func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
|
|
return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
|
|
State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
|
|
}
|
|
|
|
// backupsWait is a wait of the same provider for another part than the one its consumers need.
|
|
var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
|
|
|
|
// failingConsumer is a consumer whose check that needs the database fails.
|
|
func failingConsumer(module string) inventory.ResourceHealth {
|
|
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
|
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
|
}
|
|
|
|
// dbSecrets is what the database's waits name: own secrets issued outside the mesh, so a wait for one checks out
|
|
// while nobody gave it (ADR 0283 decision 3, novox/hq issue 450).
|
|
var dbSecrets = catalogue.OwnSecrets{
|
|
"licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside},
|
|
"backup-key": {Path: "/s/backup-key", IssuedBy: catalogue.IssuedOutside},
|
|
}
|
|
|
|
// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
|
|
// catalogue, what was given on the provider's machine (nothing), and each consumer's provider already looked up,
|
|
// as providerFor memoises it.
|
|
func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
|
|
h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
|
|
shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", OwnSecrets: dbSecrets,
|
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}},
|
|
waits: operatorWaitFacts{given: map[string]map[string]time.Time{"db@anchor": {}}}}
|
|
for k, p := range bound {
|
|
h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
|
|
}
|
|
return h
|
|
}
|
|
|
|
var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
|
|
|
|
var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
|
|
|
|
func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
|
|
return map[string]inventory.NodeHealth{
|
|
"anchor": {Node: "anchor", Resources: provider},
|
|
"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
|
|
}
|
|
}
|
|
|
|
// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
|
|
// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
|
|
// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
|
|
func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
|
|
shop := []inventory.ResourceHealth{failingConsumer("shop")}
|
|
unhealthyDB := waitingDatabase()
|
|
unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
|
healthyDB := waitingDatabase()
|
|
healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
|
|
byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
|
|
Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
|
|
byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", OwnSecrets: dbSecrets, Provides: []catalogue.Offer{{
|
|
Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
|
|
for _, c := range []struct {
|
|
name string
|
|
hold *holding
|
|
held bool
|
|
onlyWaits bool
|
|
uncovered bool
|
|
}{
|
|
{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
|
|
{"the wait is for the provision's shared credential", byCredential, true, true, false},
|
|
{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
|
|
{"only the needs-operator condition, its parts not said", holdingOf(
|
|
[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
|
|
shopFailingBeside(), shopOnTheDatabase), false, false, true},
|
|
{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
|
|
{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
|
|
} {
|
|
by, held := c.hold.heldWith("laptop", "shop", shop)
|
|
if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
|
|
t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
|
|
by.waits, c.held, c.onlyWaits)
|
|
}
|
|
if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
|
|
t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
|
|
}
|
|
}
|
|
}
|
|
|
|
// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
|
|
// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
|
|
func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
|
|
now := time.Now()
|
|
since := now.Add(-time.Minute)
|
|
for _, c := range []struct {
|
|
name string
|
|
provider inventory.ResourceHealth
|
|
want health
|
|
says []string
|
|
}{
|
|
{"a provider that only waits", waitingDatabase(), healthPerson,
|
|
[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
|
|
{"an unhealthy provider", func() inventory.ResourceHealth {
|
|
r := waitingDatabase()
|
|
r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
|
return r
|
|
}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
|
|
} {
|
|
healths := shopFailingBeside(c.provider)
|
|
for m, h := range healths {
|
|
h.HeardAt = now
|
|
healths[m] = h
|
|
}
|
|
f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
|
|
h, why := moduleHealthWord("shop", "laptop", since, f)
|
|
if h != c.want {
|
|
t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
|
|
continue
|
|
}
|
|
for _, s := range c.says {
|
|
if !strings.Contains(why, s) {
|
|
t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
|
|
// as the statements do.
|
|
func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
|
|
byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
was := readHoldingFor
|
|
t.Cleanup(func() { readHoldingFor = was })
|
|
healths := byProvider
|
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
return holdingOf(open, healths, shopOnTheDatabase), nil
|
|
}
|
|
for look := 1; look <= 2; look++ {
|
|
healths = byProvider
|
|
if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
|
|
map[string]int{"db": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for look := 1; look <= 2; look++ {
|
|
healths = byConsumer
|
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
|
|
map[string]int{"shop": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return open
|
|
}
|
|
|
|
func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
|
|
for i, c := range open {
|
|
if c.Key == key {
|
|
return &open[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
|
|
// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
|
|
// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
|
|
func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
|
|
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
|
|
provider := conditionOf(open, needsOperatorKey("db", "anchor"))
|
|
if len(open) != 1 || provider == nil {
|
|
t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
|
|
"provider's needs-operator alone", openKeysOf(open))
|
|
}
|
|
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
|
t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
|
|
}
|
|
if provider.Severity != conditions.Warning {
|
|
t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
|
|
}
|
|
if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
|
|
t.Fatalf("the provider's condition: %+v", provider)
|
|
}
|
|
}
|
|
|
|
// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
|
|
func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
backups := waitingDatabaseFor(backupsWait)
|
|
open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
|
|
consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
|
|
if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
|
|
t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
|
|
}
|
|
if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") {
|
|
t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
|
|
}
|
|
}
|
|
|
|
// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
|
|
// wait for the password.
|
|
func reloginBesideAWait() []inventory.ResourceHealth {
|
|
return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
|
|
waitingResource(passwordWait)}
|
|
}
|
|
|
|
// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
|
|
// operator owes it.
|
|
func TestAWaitBesideAReloginIsSaid(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
|
|
map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, open := needsOperatorOpen(t, k)
|
|
if got == nil {
|
|
t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
|
|
}
|
|
if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
|
|
t.Fatalf("the needs-operator beside the relogin: %+v", got)
|
|
}
|
|
relogged := false
|
|
for _, c := range open {
|
|
relogged = relogged || c.Key == reloginKey("mounts", "workstation")
|
|
}
|
|
if !relogged {
|
|
t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
|
|
}
|
|
// The login done, the wait stays said and the relogin clears.
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
|
|
map[string]int{"mounts": 3}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, open = needsOperatorOpen(t, k)
|
|
if got == nil || len(open) != 1 {
|
|
t.Fatalf("after the new login: %v", openKeysOf(open))
|
|
}
|
|
}
|
|
|
|
// (2) The same beside a directory used as found.
|
|
func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
found := foundDirectory("mounts", time.Now().Add(-time.Hour))
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
|
"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, open := needsOperatorOpen(t, k)
|
|
if got == nil {
|
|
t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
|
|
}
|
|
asFound := false
|
|
for _, c := range open {
|
|
asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
|
|
}
|
|
if !asFound {
|
|
t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
|
|
}
|
|
}
|
|
|
|
// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
|
|
// what fails.
|
|
func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
|
Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
|
"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, open := needsOperatorOpen(t, k)
|
|
if got == nil {
|
|
t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
|
|
}
|
|
var fault *conditions.Condition
|
|
for i, c := range open {
|
|
if c.Key == moduleUnhealthyKey("mounts", "workstation") {
|
|
fault = &open[i]
|
|
}
|
|
}
|
|
if fault == nil {
|
|
t.Fatalf("the fault is not said: %v", openKeysOf(open))
|
|
}
|
|
if strings.Contains(fault.Summary, "mounts.watch") {
|
|
t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
|
|
}
|
|
}
|
|
|
|
func openKeysOf(cs []conditions.Condition) []string {
|
|
var out []string
|
|
for _, c := range cs {
|
|
out = append(out, c.Key)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying
|
|
// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review).
|
|
func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
start := time.Now().Add(-time.Second)
|
|
healthy := waitingDatabase()
|
|
healthy.State, healthy.Waits = link.StateHealthy, nil
|
|
healths := shopFailingBeside(healthy)
|
|
was := readHoldingFor
|
|
t.Cleanup(func() { readHoldingFor = was })
|
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
return holdingOf(open, healths, shopOnTheDatabase), nil
|
|
}
|
|
shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil {
|
|
t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open))
|
|
}
|
|
healths = shopFailingBeside(waitingDatabase())
|
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var why string
|
|
for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); {
|
|
events, err := k.HistorySince(ctx, start)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, e := range events {
|
|
if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared {
|
|
why = e.Why
|
|
}
|
|
}
|
|
if why == "" {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
if !strings.Contains(why, "waits on db on anchor, which waits for you") {
|
|
t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why)
|
|
}
|
|
}
|