mesh-controller/cmd/mesh-controller/plan_looks_test.go

package main

import (
	"context"
	"errors"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
)

func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) {
	sent := time.Now().Add(-sentAgo).UTC()
	p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0,
		Modules: map[string]*inventory.PlanModule{
			"a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
			"b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
		}}
	r := inventory.Reported{Node: "ace", Sent: &sent}
	if reported {
		at := sent.Add(time.Minute)
		r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied
	}
	return p, []inventory.Reported{r}
}

// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR
// 0406 §5); one that reported on the send, or that is silent already, is none.
func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) {
	now := time.Now().UTC()
	p, reports := rolloutOf(5*time.Minute, false)
	waits := silentWaitsOf(p, reports, now, nil)
	if len(waits) != 1 {
		t.Fatalf("waits = %v; want the one machine", waits)
	}
	for w, at := range waits {
		if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute {
			t.Errorf("wait %+v sent at %v is wrong", w, at)
		}
	}
	p, reports = rolloutOf(5*time.Minute, true)
	if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
		t.Errorf("a machine that reported on its send is waited on: %v", waits)
	}
	p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false)
	if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
		t.Errorf("a machine silent past its time is waited on again: %v", waits)
	}
	p, reports = rolloutOf(time.Minute, false)
	p.State = inventory.PlanDone
	if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
		t.Errorf("a plan that ended is waiting: %v", waits)
	}
}

// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait.
func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) {
	since := time.Now().Add(-time.Minute).UTC()
	g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since}
	p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}}
	waits := silentWaitsOf(p, nil, time.Now(), nil)
	if len(waits) != 2 {
		t.Fatalf("waits = %v; want g14 and novox", waits)
	}
	g.Verdict = inventory.GatePassed
	if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 {
		t.Errorf("a gate with its verdict still waits: %v", waits)
	}
}

// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted.
func TestTheKeysFollowTheWaits(t *testing.T) {
	t1 := time.Now().UTC()
	t2 := t1.Add(time.Minute)
	a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"}
	arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1})
	if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) {
		t.Errorf("armed %v; want b (sent again) and c (no key)", arm)
	}
	if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) {
		t.Errorf("dropped %v; want the key with no wait", drop)
	}
	if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second {
		t.Errorf("a wait long over has a key of %v; want the least, one second", ttl)
	}
}

func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker {
	l := newLooker(func(ctx context.Context) bool {
		looks.Add(1)
		if block != nil {
			<-block
		}
		return failures.Add(-1) >= 0
	})
	l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond
	return l
}

func eventually(t *testing.T, what string, ok func() bool) {
	t.Helper()
	for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
		if ok() {
			return
		}
	}
	t.Fatalf("never: %s", what)
}

// Asks made while a look runs are not lost and are one look after it, not one each.

// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails
// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry).

func TestTheRetryWaitDoublesToItsMost(t *testing.T) {
	l := newLooker(func(context.Context) bool { return true })
	l.base, l.most = 5*time.Second, time.Minute
	var seen []time.Duration
	for i := 0; i < 6; i++ {
		l.after(true)
		seen = append(seen, l.backoff)
		l.retry.Stop()
		l.retry = nil
	}
	want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute}
	for i := range want {
		if seen[i] != want[i] {
			t.Fatalf("waits %v; want %v", seen, want)
		}
	}
}

// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known.

// The tier-sends a step made are said once each, per machine, with the modules it was sent.
func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) {
	sends := &sendsMade{}
	ctx := withSendsMade(context.Background(), sends)
	at := time.Now().UTC()
	noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at)
	noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at)
	noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept
	got := sentBodies("plan-9", sends)
	if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" ||
		got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" {
		t.Errorf("sent bodies = %+v", got)
	}
}

// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5).

// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only
// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said.

type brokenClock struct{ silentClock }

func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
	return nil, errors.New("the bus is down")
}

// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
	w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
	sent := time.Now().UTC()
	for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
		"no clock": nil} {
		t.Run(name, func(t *testing.T) {
			noteRefusedSendKeys(nil)
			t.Cleanup(func() { noteRefusedSendKeys(nil) })
			err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
			if clock != nil && err == nil {
				t.Fatal("no error given back")
			}
			got := observeRefusedSendKeys()
			if len(got) != 1 {
				t.Fatalf("observations %+v", got)
			}
			o := got[0]
			if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
				!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
				t.Fatalf("the condition does not name the plan and the machine: %+v", o)
			}
		})
	}
}

// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
	w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
	sent := time.Now().UTC()
	noteRefusedSendKeys(nil)
	t.Cleanup(func() { noteRefusedSendKeys(nil) })
	want := map[silentWait]time.Time{w: sent}
	if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
		t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
	}
	clock := &keptClock{}
	if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
		t.Fatal(err)
	}
	if !clock.armed[w].Equal(sent) {
		t.Fatalf("the next look did not write the key again: %v", clock.armed)
	}
	if got := observeRefusedSendKeys(); len(got) != 0 {
		t.Fatalf("the condition stays after the key was written: %+v", got)
	}
}

type armRefusingClock struct{ silentClock }

func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
	return errors.New("the bus refuses the key")
}

// keptClock is a clock that takes every key.
type keptClock struct {
	silentClock
	armed map[silentWait]time.Time
}

func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
	if c.armed == nil {
		c.armed = map[silentWait]time.Time{}
	}
	c.armed[w] = sent
	return nil
}

// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
// looked at: the controller follows its own plan-moved (issue 974, round 1).

// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that
// never reaches its end.
type hungClock struct{ silentClock }

func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
	select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it)
}

// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
	was := silentCallBound
	silentCallBound = 50 * time.Millisecond
	t.Cleanup(func() { silentCallBound = was })
	w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
	sent := time.Now().UTC()
	done := make(chan error, 1)
	go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
	select {
	case err := <-done:
		if err == nil {
			t.Fatal("a clock that never answered was reported as fine")
		}
	case <-time.After(5 * time.Second):
		t.Fatal("the look is still waiting for a clock that never answers")
	}
}