jschoubben
d5cf3b42fe
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery failed: its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine
376 lines
17 KiB
Go
376 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
|
|
// issue 405, found in the review of ADR 0283's controller change).
|
|
//
|
|
// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
|
|
// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
|
|
// began before it was, and its unit failed in that account's own service manager.
|
|
|
|
// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
|
|
// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
|
|
// names the part that waits by the provision it gives.
|
|
func waitingDatabase() inventory.ResourceHealth {
|
|
return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
|
|
What: "the licence key of the database"})
|
|
}
|
|
|
|
func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
|
|
return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
|
|
State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
|
|
}
|
|
|
|
// backupsWait is a wait of the same provider for another part than the one its consumers need.
|
|
var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
|
|
|
|
// failingConsumer is a consumer whose check that needs the database fails.
|
|
func failingConsumer(module string) inventory.ResourceHealth {
|
|
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
|
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
|
}
|
|
|
|
// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
|
|
// catalogue, and each consumer's provider already looked up, as providerFor memoises it.
|
|
func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
|
|
h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
|
|
shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1",
|
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}}}
|
|
for k, p := range bound {
|
|
h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
|
|
}
|
|
return h
|
|
}
|
|
|
|
var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
|
|
|
|
var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
|
|
|
|
func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
|
|
return map[string]inventory.NodeHealth{
|
|
"anchor": {Node: "anchor", Resources: provider},
|
|
"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
|
|
}
|
|
}
|
|
|
|
// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
|
|
// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
|
|
// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
|
|
func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
|
|
shop := []inventory.ResourceHealth{failingConsumer("shop")}
|
|
unhealthyDB := waitingDatabase()
|
|
unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
|
healthyDB := waitingDatabase()
|
|
healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
|
|
byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
|
|
Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
|
|
byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{
|
|
Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
|
|
for _, c := range []struct {
|
|
name string
|
|
hold *holding
|
|
held bool
|
|
onlyWaits bool
|
|
uncovered bool
|
|
}{
|
|
{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
|
|
{"the wait is for the provision's shared credential", byCredential, true, true, false},
|
|
{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
|
|
{"only the needs-operator condition, its parts not said", holdingOf(
|
|
[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
|
|
shopFailingBeside(), shopOnTheDatabase), false, false, true},
|
|
{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
|
|
{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
|
|
} {
|
|
by, held := c.hold.heldWith("laptop", "shop", shop)
|
|
if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
|
|
t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
|
|
by.waits, c.held, c.onlyWaits)
|
|
}
|
|
if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
|
|
t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
|
|
}
|
|
}
|
|
}
|
|
|
|
// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
|
|
// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
|
|
func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
|
|
now := time.Now()
|
|
since := now.Add(-time.Minute)
|
|
for _, c := range []struct {
|
|
name string
|
|
provider inventory.ResourceHealth
|
|
want health
|
|
says []string
|
|
}{
|
|
{"a provider that only waits", waitingDatabase(), healthPerson,
|
|
[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
|
|
{"an unhealthy provider", func() inventory.ResourceHealth {
|
|
r := waitingDatabase()
|
|
r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
|
|
return r
|
|
}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
|
|
} {
|
|
healths := shopFailingBeside(c.provider)
|
|
for m, h := range healths {
|
|
h.HeardAt = now
|
|
healths[m] = h
|
|
}
|
|
f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
|
|
h, why := moduleHealthWord("shop", "laptop", since, f)
|
|
if h != c.want {
|
|
t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
|
|
continue
|
|
}
|
|
for _, s := range c.says {
|
|
if !strings.Contains(why, s) {
|
|
t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
|
|
// as the statements do.
|
|
func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
|
|
byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
was := readHoldingFor
|
|
t.Cleanup(func() { readHoldingFor = was })
|
|
healths := byProvider
|
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
return holdingOf(open, healths, shopOnTheDatabase), nil
|
|
}
|
|
for look := 1; look <= 2; look++ {
|
|
healths = byProvider
|
|
if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
|
|
map[string]int{"db": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for look := 1; look <= 2; look++ {
|
|
healths = byConsumer
|
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
|
|
map[string]int{"shop": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
open, err := k.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return open
|
|
}
|
|
|
|
func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
|
|
for i, c := range open {
|
|
if c.Key == key {
|
|
return &open[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
|
|
// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
|
|
// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
|
|
func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
|
|
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
|
|
provider := conditionOf(open, needsOperatorKey("db", "anchor"))
|
|
if len(open) != 1 || provider == nil {
|
|
t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
|
|
"provider's needs-operator alone", openKeysOf(open))
|
|
}
|
|
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
|
t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
|
|
}
|
|
if provider.Severity != conditions.Warning {
|
|
t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
|
|
}
|
|
if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
|
|
t.Fatalf("the provider's condition: %+v", provider)
|
|
}
|
|
}
|
|
|
|
// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
|
|
func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
backups := waitingDatabaseFor(backupsWait)
|
|
open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
|
|
consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
|
|
if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
|
|
t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
|
|
}
|
|
if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") {
|
|
t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
|
|
}
|
|
}
|
|
|
|
// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
|
|
// wait for the password.
|
|
func reloginBesideAWait() []inventory.ResourceHealth {
|
|
return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
|
|
waitingResource(passwordWait)}
|
|
}
|
|
|
|
// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
|
|
// operator owes it.
|
|
func TestAWaitBesideAReloginIsSaid(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
|
|
map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, open := needsOperatorOpen(t, k)
|
|
if got == nil {
|
|
t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
|
|
}
|
|
if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
|
|
t.Fatalf("the needs-operator beside the relogin: %+v", got)
|
|
}
|
|
relogged := false
|
|
for _, c := range open {
|
|
relogged = relogged || c.Key == reloginKey("mounts", "workstation")
|
|
}
|
|
if !relogged {
|
|
t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
|
|
}
|
|
// The login done, the wait stays said and the relogin clears.
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
|
|
map[string]int{"mounts": 3}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, open = needsOperatorOpen(t, k)
|
|
if got == nil || len(open) != 1 {
|
|
t.Fatalf("after the new login: %v", openKeysOf(open))
|
|
}
|
|
}
|
|
|
|
// (2) The same beside a directory used as found.
|
|
func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
found := foundDirectory("mounts", time.Now().Add(-time.Hour))
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
|
"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, open := needsOperatorOpen(t, k)
|
|
if got == nil {
|
|
t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
|
|
}
|
|
asFound := false
|
|
for _, c := range open {
|
|
asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
|
|
}
|
|
if !asFound {
|
|
t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
|
|
}
|
|
}
|
|
|
|
// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
|
|
// what fails.
|
|
func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
|
|
Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
|
|
"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
got, open := needsOperatorOpen(t, k)
|
|
if got == nil {
|
|
t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
|
|
}
|
|
var fault *conditions.Condition
|
|
for i, c := range open {
|
|
if c.Key == moduleUnhealthyKey("mounts", "workstation") {
|
|
fault = &open[i]
|
|
}
|
|
}
|
|
if fault == nil {
|
|
t.Fatalf("the fault is not said: %v", openKeysOf(open))
|
|
}
|
|
if strings.Contains(fault.Summary, "mounts.watch") {
|
|
t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
|
|
}
|
|
}
|
|
|
|
func openKeysOf(cs []conditions.Condition) []string {
|
|
var out []string
|
|
for _, c := range cs {
|
|
out = append(out, c.Key)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying
|
|
// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review).
|
|
func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) {
|
|
k, _ := withConditionsInMemory(t)
|
|
ctx := t.Context()
|
|
start := time.Now().Add(-time.Second)
|
|
healthy := waitingDatabase()
|
|
healthy.State, healthy.Waits = link.StateHealthy, nil
|
|
healths := shopFailingBeside(healthy)
|
|
was := readHoldingFor
|
|
t.Cleanup(func() { readHoldingFor = was })
|
|
readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
|
|
return holdingOf(open, healths, shopOnTheDatabase), nil
|
|
}
|
|
shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}
|
|
for look := 1; look <= 2; look++ {
|
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil {
|
|
t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open))
|
|
}
|
|
healths = shopFailingBeside(waitingDatabase())
|
|
if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var why string
|
|
for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); {
|
|
events, err := k.HistorySince(ctx, start)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, e := range events {
|
|
if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared {
|
|
why = e.Why
|
|
}
|
|
}
|
|
if why == "" {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
if !strings.Contains(why, "waits on db on anchor, which waits for you") {
|
|
t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why)
|
|
}
|
|
}
|