package main import ( "context" "errors" "strings" "sync/atomic" "testing" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) { sent := time.Now().Add(-sentAgo).UTC() p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0, Modules: map[string]*inventory.PlanModule{ "a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, "b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, }} r := inventory.Reported{Node: "ace", Sent: &sent} if reported { at := sent.Add(time.Minute) r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied } return p, []inventory.Reported{r} } // A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR // 0406 §5); one that reported on the send, or that is silent already, is none. func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) { now := time.Now().UTC() p, reports := rolloutOf(5*time.Minute, false) waits := silentWaitsOf(p, reports, now, nil) if len(waits) != 1 { t.Fatalf("waits = %v; want the one machine", waits) } for w, at := range waits { if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute { t.Errorf("wait %+v sent at %v is wrong", w, at) } } p, reports = rolloutOf(5*time.Minute, true) if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { t.Errorf("a machine that reported on its send is waited on: %v", waits) } p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false) if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { t.Errorf("a machine silent past its time is waited on again: %v", waits) } p, reports = rolloutOf(time.Minute, false) p.State = inventory.PlanDone if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { t.Errorf("a plan that ended is waiting: %v", waits) } } // A release's open gate waits on its machines from when it began judging, and its verdict ends the wait. func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) { since := time.Now().Add(-time.Minute).UTC() g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since} p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}} waits := silentWaitsOf(p, nil, time.Now(), nil) if len(waits) != 2 { t.Fatalf("waits = %v; want g14 and novox", waits) } g.Verdict = inventory.GatePassed if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 { t.Errorf("a gate with its verdict still waits: %v", waits) } } // The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted. func TestTheKeysFollowTheWaits(t *testing.T) { t1 := time.Now().UTC() t2 := t1.Add(time.Minute) a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"} arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1}) if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) { t.Errorf("armed %v; want b (sent again) and c (no key)", arm) } if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) { t.Errorf("dropped %v; want the key with no wait", drop) } if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second { t.Errorf("a wait long over has a key of %v; want the least, one second", ttl) } } func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker { l := newLooker(func(ctx context.Context) bool { looks.Add(1) if block != nil { <-block } return failures.Add(-1) >= 0 }) l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond return l } func eventually(t *testing.T, what string, ok func() bool) { t.Helper() for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { if ok() { return } } t.Fatalf("never: %s", what) } // Asks made while a look runs are not lost and are one look after it, not one each. // A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails // nothing ends the retries (novox/hq ADR 0406 §5; the one named retry). func TestTheRetryWaitDoublesToItsMost(t *testing.T) { l := newLooker(func(context.Context) bool { return true }) l.base, l.most = 5*time.Second, time.Minute var seen []time.Duration for i := 0; i < 6; i++ { l.after(true) seen = append(seen, l.backoff) l.retry.Stop() l.retry = nil } want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute} for i := range want { if seen[i] != want[i] { t.Fatalf("waits %v; want %v", seen, want) } } } // A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known. // The tier-sends a step made are said once each, per machine, with the modules it was sent. func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { sends := &sendsMade{} ctx := withSendsMade(context.Background(), sends) at := time.Now().UTC() noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at) noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at) noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept got := sentBodies("plan-9", sends) if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" || got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" { t.Errorf("sent bodies = %+v", got) } } // A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5). // A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only // once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said. type brokenClock struct{ silentClock } func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, errors.New("the bus is down") } // 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by // nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error // is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4). func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1 w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} sent := time.Now().UTC() for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{}, "no clock": nil} { t.Run(name, func(t *testing.T) { noteRefusedSendKeys(nil) t.Cleanup(func() { noteRefusedSendKeys(nil) }) err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent) if clock != nil && err == nil { t.Fatal("no error given back") } got := observeRefusedSendKeys() if len(got) != 1 { t.Fatalf("observations %+v", got) } o := got[0] if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused || !strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") { t.Fatalf("the condition does not name the plan and the machine: %+v", o) } }) } } // 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is. func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2 w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} sent := time.Now().UTC() noteRefusedSendKeys(nil) t.Cleanup(func() { noteRefusedSendKeys(nil) }) want := map[silentWait]time.Time{w: sent} if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 { t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys()) } clock := &keptClock{} if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil { t.Fatal(err) } if !clock.armed[w].Equal(sent) { t.Fatalf("the next look did not write the key again: %v", clock.armed) } if got := observeRefusedSendKeys(); len(got) != 0 { t.Fatalf("the condition stays after the key was written: %+v", got) } } type armRefusingClock struct{ silentClock } func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil } func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error { return errors.New("the bus refuses the key") } // keptClock is a clock that takes every key. type keptClock struct { silentClock armed map[silentWait]time.Time } func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil } func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error { if c.armed == nil { c.armed = map[silentWait]time.Time{} } c.armed[w] = sent return nil } // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is // looked at: the controller follows its own plan-moved (issue 974, round 1). // A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that // never reaches its end. type hungClock struct{ silentClock } func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) { select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it) } // issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans // and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition. func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) { was := silentCallBound silentCallBound = 50 * time.Millisecond t.Cleanup(func() { silentCallBound = was }) w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"} sent := time.Now().UTC() done := make(chan error, 1) go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }() select { case err := <-done: if err == nil { t.Fatal("a clock that never answered was reported as fine") } case <-time.After(5 * time.Second): t.Fatal("the look is still waiting for a clock that never answers") } }