Keep a waiting provider's hold within ADR 0283 (issue 405 review)

A consumer held under a provider that only waits for the operator now
passes its gate as a wait for a person, carrying the wait, so no walk
waits on the operator's secret. Only consumers of the waiting part are
held; one that cannot be matched is raised on its own and says its
provider waits. needs-operator stays a warning while consumers wait.

cmd/mesh-controller/waiting_provider_test.go

@@ -20,23 +20,33 @@ import (
 // began before it was, and its unit failed in that account's own service manager.
 
 // waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
-// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2).
+// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait
+// names the part that waits by the provision it gives.
 func waitingDatabase() inventory.ResourceHealth {
-	return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
-		State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence",
-		Waits: []inventory.Wait{{Part: "the server", Secret: "licence", What: "the licence key of the database"}}}
+	return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence",
+		What: "the licence key of the database"})
 }
 
+func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
+		State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits}
+}
+
+// backupsWait is a wait of the same provider for another part than the one its consumers need.
+var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"}
+
 // failingConsumer is a consumer whose check that needs the database fails.
 func failingConsumer(module string) inventory.ResourceHealth {
 	return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
 		State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
 }
 
-// holdingOf is one reading of the record without a store: the newest statements, the open conditions, and each
-// consumer's provider already looked up, as providerFor memoises it.
+// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the
+// catalogue, and each consumer's provider already looked up, as providerFor memoises it.
 func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
-	h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{}}
+	h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{},
+		shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1",
+			Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}}}
 	for k, p := range bound {
 		h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
 	}
@@ -45,58 +55,114 @@ func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHea
 
 var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
 
-// (1) A provider whose only part not healthy waits for the operator holds the findings of its consumers under its
-// needs-operator condition, as ADR 0240 rule 5 holds them under an unhealthy one.
-func TestAProviderWaitingForTheOperatorHoldsItsConsumers(t *testing.T) {
-	healths := map[string]inventory.NodeHealth{
-		"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}},
+var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
+
+func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth {
+	return map[string]inventory.NodeHealth{
+		"anchor": {Node: "anchor", Resources: provider},
 		"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
 	}
-	bound := map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
-	hold := holdingOf(nil, healths, bound)
-	p, held := hold.heldUnder("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
-	if !held || p != theDatabase {
-		t.Fatalf("a consumer of a provider waiting for the operator is not held under it: %v %v", p, held)
-	}
-	if got := hold.waitersOn(theDatabase); len(got) != 1 || got[0] != "shop on laptop" {
-		t.Fatalf("the waiting provider lists %v as waiting on it; want shop on laptop", got)
-	}
-	// Its needs-operator condition open is enough, as its unhealthy one is.
-	open := []conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}}
-	healths["anchor"] = inventory.NodeHealth{Node: "anchor"}
-	if _, held := holdingOf(open, healths, bound).heldUnder("laptop", "shop",
-		[]inventory.ResourceHealth{failingConsumer("shop")}); !held {
-		t.Fatal("a consumer is not held under its provider's open needs-operator condition")
-	}
-	// A provider healthy holds nothing: what the consumer finds is its own.
-	healthy := waitingDatabase()
-	healthy.State, healthy.Waits = link.StateHealthy, nil
-	healths["anchor"] = inventory.NodeHealth{Node: "anchor", Resources: []inventory.ResourceHealth{healthy}}
-	if _, held := holdingOf(nil, healths, bound).heldUnder("laptop", "shop",
-		[]inventory.ResourceHealth{failingConsumer("shop")}); held {
-		t.Fatal("a consumer of a healthy provider is held")
-	}
 }
 
-// (1) The consumer raises nothing of its own, and the provider's needs-operator condition says who waits on it.
-func TestAWaitingProvidersConditionListsWhoWaitsOnIt(t *testing.T) {
-	k, _ := withConditionsInMemory(t)
-	ctx := t.Context()
-	healths := map[string]inventory.NodeHealth{
-		"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}},
-		"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
+// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the
+// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is
+// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong).
+func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) {
+	shop := []inventory.ResourceHealth{failingConsumer("shop")}
+	unhealthyDB := waitingDatabase()
+	unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused"
+	healthyDB := waitingDatabase()
+	healthyDB.State, healthyDB.Waits = link.StateHealthy, nil
+	byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server",
+		Secret: "licence", What: "the licence key"})), shopOnTheDatabase)
+	byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{
+		Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}}
+	for _, c := range []struct {
+		name      string
+		hold      *holding
+		held      bool
+		onlyWaits bool
+		uncovered bool
+	}{
+		{"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false},
+		{"the wait is for the provision's shared credential", byCredential, true, true, false},
+		{"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true},
+		{"only the needs-operator condition, its parts not said", holdingOf(
+			[]conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}},
+			shopFailingBeside(), shopOnTheDatabase), false, false, true},
+		{"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false},
+		{"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false},
+	} {
+		by, held := c.hold.heldWith("laptop", "shop", shop)
+		if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits {
+			t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider,
+				by.waits, c.held, c.onlyWaits)
+		}
+		if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered {
+			t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered)
+		}
 	}
-	bound := map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
+}
+
+// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a
+// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before.
+func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) {
+	now := time.Now()
+	since := now.Add(-time.Minute)
+	for _, c := range []struct {
+		name     string
+		provider inventory.ResourceHealth
+		want     health
+		says     []string
+	}{
+		{"a provider that only waits", waitingDatabase(), healthPerson,
+			[]string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}},
+		{"an unhealthy provider", func() inventory.ResourceHealth {
+			r := waitingDatabase()
+			r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused"
+			return r
+		}(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}},
+	} {
+		healths := shopFailingBeside(c.provider)
+		for m, h := range healths {
+			h.HeardAt = now
+			healths[m] = h
+		}
+		f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))}
+		h, why := moduleHealthWord("shop", "laptop", since, f)
+		if h != c.want {
+			t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want)
+			continue
+		}
+		for _, s := range c.says {
+			if !strings.Contains(why, s) {
+				t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s)
+			}
+		}
+	}
+}
+
+// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes
+// as the statements do.
+func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth,
+	byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition {
+	t.Helper()
+	ctx := t.Context()
 	was := readHoldingFor
 	t.Cleanup(func() { readHoldingFor = was })
+	healths := byProvider
 	readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
-		return holdingOf(open, healths, bound), nil
+		return holdingOf(open, healths, shopOnTheDatabase), nil
 	}
 	for look := 1; look <= 2; look++ {
-		if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": {waitingDatabase()}},
+		healths = byProvider
+		if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider},
 			map[string]int{"db": look}, time.Now()); err != nil {
 			t.Fatal(err)
 		}
+	}
+	for look := 1; look <= 2; look++ {
+		healths = byConsumer
 		if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
 			map[string]int{"shop": look}, time.Now()); err != nil {
 			t.Fatal(err)
@@ -106,27 +172,55 @@ func TestAWaitingProvidersConditionListsWhoWaitsOnIt(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	var keys []string
-	var provider *conditions.Condition
+	return open
+}
+
+func conditionOf(open []conditions.Condition, key string) *conditions.Condition {
 	for i, c := range open {
-		keys = append(keys, c.Key)
-		if c.Key == needsOperatorKey("db", "anchor") {
-			provider = &open[i]
+		if c.Key == key {
+			return &open[i]
 		}
 	}
+	return nil
+}
+
+// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and
+// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's,
+// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved.
+func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()},
+		map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase()))
+	provider := conditionOf(open, needsOperatorKey("db", "anchor"))
 	if len(open) != 1 || provider == nil {
-		t.Fatalf("a provider waiting for the operator and a consumer failing on it raised %v; want the provider's "+
-			"needs-operator alone", keys)
+		t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+
+			"provider's needs-operator alone", openKeysOf(open))
 	}
 	if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
 		t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
 	}
-	// Its words still name the act: it is the operator's, whoever waits on it.
+	if provider.Severity != conditions.Warning {
+		t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity)
+	}
 	if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
 		t.Fatalf("the provider's condition: %+v", provider)
 	}
 }
 
+// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits.
+func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	backups := waitingDatabaseFor(backupsWait)
+	open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups))
+	consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop"))
+	if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil {
+		t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open))
+	}
+	if said := consumer.Evidence[0].Said; !strings.Contains(said, "its provider db on anchor waits for the operator") {
+		t.Fatalf("the consumer's condition does not say its provider waits: %s", said)
+	}
+}
+
 // relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
 // wait for the password.
 func reloginBesideAWait() []inventory.ResourceHealth {