Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318)
This commit is contained in:
jochen
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254,
|
||||
// issue 318).
|
||||
|
||||
func relogin(module, account string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account,
|
||||
Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module +
|
||||
", and its running session began before it was; log out of every session and in again, or reboot"}
|
||||
}
|
||||
|
||||
func userUnit(module, account string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit,
|
||||
Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
|
||||
Reason: "failed in the account's own service manager (exit-code)"}
|
||||
}
|
||||
|
||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||
// before, so that a fault is never excused.
|
||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
notInGroup := relogin("lights", "operator")
|
||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||
systemUnit := userUnit("lights", "operator")
|
||||
systemUnit.Account = ""
|
||||
otherAccount := userUnit("lights", "guest")
|
||||
container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
|
||||
State: link.StateUnhealthy, Reason: "down"}
|
||||
otherModule := userUnit("sound", "operator")
|
||||
starting := userUnit("lights", "operator")
|
||||
starting.State = link.StateStarting
|
||||
byTarget := relogin("lights", "operator")
|
||||
byTarget.Account = "" // an account said by its target alone is still that account
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
rs []inventory.ResourceHealth
|
||||
waits bool
|
||||
}{
|
||||
{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
|
||||
{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
|
||||
userUnit("lights", "operator")}, true},
|
||||
{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
|
||||
{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
|
||||
{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
|
||||
{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
|
||||
{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
|
||||
{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
|
||||
{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
|
||||
{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
|
||||
} {
|
||||
said, waits := personWait("lights", "laptop", c.rs)
|
||||
if waits != c.waits {
|
||||
t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
|
||||
continue
|
||||
}
|
||||
if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
|
||||
t.Errorf("%s: said %q", c.name, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||
// with a container down beside it is not yet healthy, as before.
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
|
||||
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
h := f.health["laptop"]
|
||||
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
|
||||
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
|
||||
f.health["laptop"] = h
|
||||
if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet {
|
||||
t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
at := h0
|
||||
say := func(rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
asLink := func(r inventory.ResourceHealth) link.ResourceHealth {
|
||||
return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State,
|
||||
Reason: r.Reason, Account: r.Account}
|
||||
}
|
||||
account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator"))
|
||||
openNow := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
say(account, unit)
|
||||
say(account, unit)
|
||||
list := openNow()
|
||||
if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" {
|
||||
t.Fatalf("raised %+v; want lights' relogin-needed alone", list)
|
||||
}
|
||||
c := list[0]
|
||||
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator ||
|
||||
!strings.HasPrefix(c.Summary, "relogin needed on laptop:") {
|
||||
t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary)
|
||||
}
|
||||
// After the new login the account is healthy, and the unit, still failing, is the module's own fault.
|
||||
account.State, account.Reason = link.StateHealthy, ""
|
||||
say(account, unit)
|
||||
list = openNow()
|
||||
if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" {
|
||||
t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list)
|
||||
}
|
||||
unit.State, unit.Reason = link.StateHealthy, ""
|
||||
say(account, unit)
|
||||
if list = openNow(); len(list) != 0 {
|
||||
t.Fatalf("after the unit runs: %+v; want nothing open", list)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
||||
Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy,
|
||||
Reason: "down"}}}}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateBound = 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
||||
!strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed {
|
||||
t.Fatal("late's build is not marked failed at its gate")
|
||||
}
|
||||
current, _ := inv.CurrentBuilds(ctx)
|
||||
if current["app"].Commit != "c2" || current["late"].Commit != "c1" {
|
||||
t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit,
|
||||
current["late"].Commit)
|
||||
}
|
||||
if !strings.Contains(p.Note, "passed on their own and kept: app") {
|
||||
t.Errorf("the walk does not say what kept its pass: %s", p.Note)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user