Raise a module's unhealthy urgent when its manifest says why (hq ADR 0355, issue 691)

A broken service people rely on, such as mail, was said only as a warning on
the desktop. A module may now declare unhealthy urgent with what people lose;
the module check refuses it without why or with another severity.

cmd/mesh-controller/module_health.go

@@ -240,6 +240,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 			continue
 		}
 		o := moduleUnhealthyObservation(m, node, unhealthy[m])
+		declaredSeverity(&o, hold, m)
 		if hold != nil {
 			if by, held := hold.heldWith(node, m, unhealthy[m]); held {
 				// Held under the provider's condition: nothing of its own, and the provider's says it waits. The
@@ -347,6 +348,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
 	} else {
 		_, rest := besideAWait(p.Module, rs)
 		o = moduleUnhealthyObservation(p.Module, p.Node, rest)
+		declaredSeverity(&o, hold, p.Module)
 		if said, waits := personWait(p.Module, p.Node, rest); waits {
 			o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
 		}
@@ -481,6 +483,19 @@ func sayWaitingOn(o *conditions.Observation, waiters []string) {
 	o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
 }
 
+// declaredSeverity raises a module's unhealthy condition urgent when its manifest declares `unhealthy` urgent
+// (novox/hq ADR 0355 §1). Only that condition changes (ADR 0355 §4): a wait for a new login, a module that needs
+// the operator and a directory used as found keep their severities, and without the field, or without a manifest
+// to read it from, it stays the warning it was.
+func declaredSeverity(o *conditions.Observation, hold *holding, module string) {
+	if o.Kind != kindModuleUnhealthy || hold == nil {
+		return
+	}
+	if m, ok := hold.manifestOf(module); ok && m.UnhealthyIsUrgent() {
+		o.Severity = conditions.Urgent
+	}
+}
+
 // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
 // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {

cmd/mesh-controller/urgent_unhealthy_test.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A module may declare its unhealthy urgent (novox/hq ADR 0355): its module-unhealthy condition is then raised
+// urgent; without the field it stays a warning; a wait for a new login stays a warning whatever the module says.
+
+// judgedTwice judges one machine's statement twice, over a catalogue in which mail declares its unhealthy
+// urgent and notes says nothing, and returns what is open.
+func judgedTwice(t *testing.T, unhealthy map[string][]inventory.ResourceHealth) []conditions.Condition {
+	t.Helper()
+	ctx := t.Context()
+	k, _ := withConditionsInMemory(t)
+	was := readHoldingFor
+	t.Cleanup(func() { readHoldingFor = was })
+	readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
+		return &holding{ctx: context.Background(), open: open, providers: map[string]providerLookup{},
+			shelf: map[string]catalogue.Manifest{
+				"mail": {Module: "mail", Version: "1", Unhealthy: &catalogue.UnhealthyPolicy{
+					Severity: catalogue.UnhealthyUrgent, Why: "mail is refused or stuck for everyone with a mailbox"}},
+				"notes": {Module: "notes", Version: "1"},
+			}}, nil
+	}
+	streaks := map[string]int{}
+	for look := 1; look <= 2; look++ {
+		for m := range unhealthy {
+			streaks[m] = look
+		}
+		if err := judgeModuleHealth(ctx, nil, k, "anchor", unhealthy, streaks, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	open, err := k.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return open
+}
+
+func containerDown(module string) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
+		State: link.StateUnhealthy, Reason: "down"}
+}
+
+func TestAModuleThatDeclaresItsUnhealthyUrgentIsRaisedUrgent(t *testing.T) {
+	open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {containerDown("mail")}})
+	c := conditionOf(open, "module.mail.anchor.unhealthy")
+	if c == nil {
+		t.Fatalf("mail's unhealthy condition was not raised: %+v", open)
+	}
+	if c.Severity != conditions.Urgent {
+		t.Errorf("mail declares its unhealthy urgent, and it was raised %s", c.Severity)
+	}
+}
+
+func TestAModuleThatSaysNothingOfItsUnhealthyStaysAWarning(t *testing.T) {
+	open := judgedTwice(t, map[string][]inventory.ResourceHealth{"notes": {containerDown("notes")}})
+	c := conditionOf(open, "module.notes.anchor.unhealthy")
+	if c == nil {
+		t.Fatalf("notes' unhealthy condition was not raised: %+v", open)
+	}
+	if c.Severity != conditions.Warning {
+		t.Errorf("notes says nothing of its unhealthy, and it was raised %s", c.Severity)
+	}
+}
+
+// Only the module-unhealthy condition changes (ADR 0355 §4): a wait for a new login (ADR 0254) of a module that
+// declares its unhealthy urgent stays a warning.
+func TestAWaitForANewLoginStaysAWarningWhenTheModuleDeclaresItsUnhealthyUrgent(t *testing.T) {
+	open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {relogin("mail", "operator"),
+		userUnit("mail", "operator")}})
+	c := conditionOf(open, "module.mail.anchor.relogin-needed")
+	if c == nil {
+		t.Fatalf("mail's relogin-needed condition was not raised: %+v", open)
+	}
+	if c.Severity != conditions.Warning {
+		t.Errorf("a wait for a new login was raised %s; want a warning", c.Severity)
+	}
+	if conditionOf(open, "module.mail.anchor.unhealthy") != nil {
+		t.Errorf("a wait for a new login raised mail's unhealthy condition beside it: %+v", open)
+	}
+}