A refused send key raises a condition in place of the timer that stood in for it (issue 1157)

cmd/mesh-controller/send_key_refused.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and
+// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes
+// the key again; the look that writes it leaves it out of the set, and the condition clears.
+const (
+	kindSendKeyRefused = "send-key-refused"
+	sendKeyRefusedID   = "DL"
+)
+
+var refusedSendKeys = struct {
+	mu sync.Mutex
+	by map[silentWait]error
+}{}
+
+// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found.
+func noteRefusedSendKeys(refused map[silentWait]error) {
+	refusedSendKeys.mu.Lock()
+	defer refusedSendKeys.mu.Unlock()
+	refusedSendKeys.by = refused
+}
+
+// observeRefusedSendKeys is one condition per plan and machine whose send key was refused.
+func observeRefusedSendKeys() []conditions.Observation {
+	refusedSendKeys.mu.Lock()
+	waits := make([]silentWait, 0, len(refusedSendKeys.by))
+	why := map[silentWait]error{}
+	for w, err := range refusedSendKeys.by {
+		waits = append(waits, w)
+		why[w] = err
+	}
+	refusedSendKeys.mu.Unlock()
+	sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) })
+	seen := map[[2]string]bool{}
+	var out []conditions.Observation
+	for _, w := range waits {
+		if seen[[2]string{w.Plan, w.Node}] {
+			continue // a plan sends a machine one tier at a time; one condition says it
+		}
+		seen[[2]string{w.Plan, w.Node}] = true
+		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node,
+			Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning,
+			Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+
+				"names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node),
+			Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])})
+	}
+	return out
+}
+
+// probeSendKeyRefused reads what the newest look of the plans could not write.
+func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) {
+	return observeRefusedSendKeys(), nil
+}