A refused send key raises a condition in place of the timer that stood in for it (issue 1157)

cmd/mesh-controller/plan_looks_test.go

@@ -3,10 +3,12 @@ package main
 import (
 	"context"
 	"errors"
+	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -159,22 +161,76 @@ func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
 	return nil, errors.New("the bus is down")
 }
 
-// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
-// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
-func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
+// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
+// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
+// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
+func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
 	w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
 	sent := time.Now().UTC()
-	t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
-	err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
-	if err == nil {
-		t.Fatal("no error given back")
+	for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
+		"no clock": nil} {
+		t.Run(name, func(t *testing.T) {
+			noteRefusedSendKeys(nil)
+			t.Cleanup(func() { noteRefusedSendKeys(nil) })
+			err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
+			if clock != nil && err == nil {
+				t.Fatal("no error given back")
+			}
+			got := observeRefusedSendKeys()
+			if len(got) != 1 {
+				t.Fatalf("observations %+v", got)
+			}
+			o := got[0]
+			if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
+				!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
+				t.Fatalf("the condition does not name the plan and the machine: %+v", o)
+			}
+		})
 	}
-	silentTimers.mu.Lock()
-	armed := silentTimers.timers[w].Equal(sent)
-	silentTimers.mu.Unlock()
-	if !armed {
-		t.Error("no timer of its own watches the wait")
+}
+
+// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
+func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
+	w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
+	sent := time.Now().UTC()
+	noteRefusedSendKeys(nil)
+	t.Cleanup(func() { noteRefusedSendKeys(nil) })
+	want := map[silentWait]time.Time{w: sent}
+	if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
+		t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
 	}
+	clock := &keptClock{}
+	if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
+		t.Fatal(err)
+	}
+	if !clock.armed[w].Equal(sent) {
+		t.Fatalf("the next look did not write the key again: %v", clock.armed)
+	}
+	if got := observeRefusedSendKeys(); len(got) != 0 {
+		t.Fatalf("the condition stays after the key was written: %+v", got)
+	}
+}
+
+type armRefusingClock struct{ silentClock }
+
+func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
+func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
+	return errors.New("the bus refuses the key")
+}
+
+// keptClock is a clock that takes every key.
+type keptClock struct {
+	silentClock
+	armed map[silentWait]time.Time
+}
+
+func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
+func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
+	if c.armed == nil {
+		c.armed = map[silentWait]time.Time{}
+	}
+	c.armed[w] = sent
+	return nil
 }
 
 // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
@@ -189,14 +245,13 @@ func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
 }
 
 // issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
-// and every node for good. Every call to the clock has a bound, and the wait is looked at by a timer of its own.
+// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
 func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
 	was := silentCallBound
 	silentCallBound = 50 * time.Millisecond
 	t.Cleanup(func() { silentCallBound = was })
 	w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
 	sent := time.Now().UTC()
-	t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
 	done := make(chan error, 1)
 	go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
 	select {