jochen
a63939160e
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk
313 lines
15 KiB
Go
313 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254,
|
|
// issue 318).
|
|
|
|
func relogin(module, account string) inventory.ResourceHealth {
|
|
return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account,
|
|
Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module +
|
|
", and its running session began before it was; log out of every session and in again, or reboot"}
|
|
}
|
|
|
|
func userUnit(module, account string) inventory.ResourceHealth {
|
|
return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit,
|
|
Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
|
|
Reason: "failed in the account's own service manager (exit-code)"}
|
|
}
|
|
|
|
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
|
// before, so that a fault is never excused.
|
|
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
|
notInGroup := relogin("lights", "operator")
|
|
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
|
systemUnit := userUnit("lights", "operator")
|
|
systemUnit.Account = ""
|
|
otherAccount := userUnit("lights", "guest")
|
|
container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
|
|
State: link.StateUnhealthy, Reason: "down"}
|
|
otherModule := userUnit("sound", "operator")
|
|
starting := userUnit("lights", "operator")
|
|
starting.State = link.StateStarting
|
|
byTarget := relogin("lights", "operator")
|
|
byTarget.Account = "" // an account said by its target alone is still that account
|
|
|
|
for _, c := range []struct {
|
|
name string
|
|
rs []inventory.ResourceHealth
|
|
waits bool
|
|
}{
|
|
{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
|
|
{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
|
|
userUnit("lights", "operator")}, true},
|
|
{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
|
|
{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
|
|
{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
|
|
{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
|
|
{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
|
|
{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
|
|
{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
|
|
{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
|
|
} {
|
|
said, waits := personWait("lights", "laptop", c.rs)
|
|
if waits != c.waits {
|
|
t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
|
|
continue
|
|
}
|
|
if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
|
|
t.Errorf("%s: said %q", c.name, said)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
|
// with a container down beside it is not yet healthy, as before.
|
|
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
|
now := time.Now()
|
|
since := now.Add(-time.Minute)
|
|
account := relogin("lights", "operator")
|
|
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
|
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}},
|
|
groupsAdded: map[string]bool{"lights": true}}
|
|
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
|
|
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
|
|
}
|
|
// **Only a move that put the account in a new group is excused** (issue 318 review): a later build that
|
|
// adds no group did not bring the wait, and is judged as before.
|
|
f.groupsAdded["lights"] = false
|
|
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
|
|
t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why)
|
|
}
|
|
f.groupsAdded["lights"] = true
|
|
h := f.health["laptop"]
|
|
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
|
|
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
|
|
f.health["laptop"] = h
|
|
if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet {
|
|
t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why)
|
|
}
|
|
}
|
|
|
|
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
|
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
|
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
k, _ := withConditionsInMemory(t)
|
|
at := h0
|
|
say := func(rs ...link.ResourceHealth) {
|
|
t.Helper()
|
|
at = at.Add(time.Second)
|
|
for i := range rs {
|
|
rs[i].Since = at
|
|
}
|
|
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
|
Resources: rs}, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
asLink := func(r inventory.ResourceHealth) link.ResourceHealth {
|
|
return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State,
|
|
Reason: r.Reason, Account: r.Account}
|
|
}
|
|
account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator"))
|
|
openNow := func() []conditions.Condition {
|
|
t.Helper()
|
|
list, err := k.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return list
|
|
}
|
|
say(account, unit)
|
|
say(account, unit)
|
|
list := openNow()
|
|
if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" {
|
|
t.Fatalf("raised %+v; want lights' relogin-needed alone", list)
|
|
}
|
|
c := list[0]
|
|
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator ||
|
|
!strings.HasPrefix(c.Summary, "relogin needed on laptop:") {
|
|
t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary)
|
|
}
|
|
// After the new login the account is healthy, and the unit, still failing, is the module's own fault.
|
|
account.State, account.Reason = link.StateHealthy, ""
|
|
say(account, unit)
|
|
list = openNow()
|
|
if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" {
|
|
t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list)
|
|
}
|
|
unit.State, unit.Reason = link.StateHealthy, ""
|
|
say(account, unit)
|
|
if list = openNow(); len(list) != 0 {
|
|
t.Fatalf("after the unit runs: %+v; want nothing open", list)
|
|
}
|
|
}
|
|
|
|
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
|
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
|
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
|
return map[string]bool{"laptop": true}, nil
|
|
}
|
|
backlogFacts := gatherGateFacts
|
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
f, err := backlogFacts(ctx, open, component)
|
|
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
|
|
Resources: []inventory.ResourceHealth{
|
|
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
|
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy,
|
|
Reason: "down"}}}}
|
|
return f, err
|
|
}
|
|
gateEvery, gateBound = 0, 300*time.Millisecond
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
advancePlans(ctx, b.open)
|
|
if p := b.release(t); p.State != inventory.PlanRolling {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanFailed {
|
|
t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note)
|
|
}
|
|
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
|
!strings.Contains(v.Why, "on its own") {
|
|
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
|
}
|
|
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed {
|
|
t.Fatal("late's build is not marked failed at its gate")
|
|
}
|
|
current, _ := inv.CurrentBuilds(ctx)
|
|
if current["app"].Commit != "c2" || current["late"].Commit != "c1" {
|
|
t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit,
|
|
current["late"].Commit)
|
|
}
|
|
if !strings.Contains(p.Note, "passed on their own and kept: app") {
|
|
t.Errorf("the walk does not say what kept its pass: %s", p.Note)
|
|
}
|
|
}
|
|
|
|
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
|
// no button, since nothing but their own new login can do it.
|
|
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
|
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
|
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
|
plainExample(t, o, "openrazer waits for a new login on g14",
|
|
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
|
|
"group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+
|
|
"update is in place and nothing was undone.")
|
|
if len(o.Actions) != 0 || o.Needs == "" {
|
|
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
|
|
}
|
|
// **Not "your account" when it is not the operator's** (issue 318 review).
|
|
other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
|
relogin("openrazer", "guest"), userUnit("openrazer", "guest")})
|
|
plainExample(t, other, "openrazer waits for a new login on g14",
|
|
"Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+
|
|
"put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+
|
|
"until it logs in again. Its update is in place and nothing was undone.")
|
|
if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") {
|
|
t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation)
|
|
}
|
|
// The kind's own wording, for a condition raised without words, names the module whole, dots and all,
|
|
// and never claims the account is the operator's.
|
|
w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"})
|
|
if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 ||
|
|
strings.Contains(w.Explanation, "your account") {
|
|
t.Errorf("the kind's wording: %+v", w)
|
|
}
|
|
}
|
|
|
|
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
|
// became, never that it works again (issue 318 review).
|
|
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
store := conditions.NewInMemory()
|
|
told := &conditions.Told{}
|
|
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told})
|
|
t.Cleanup(func() { k.Close(context.Background()) })
|
|
at := h0
|
|
say := func(rs ...link.ResourceHealth) {
|
|
t.Helper()
|
|
at = at.Add(time.Second)
|
|
for i := range rs {
|
|
rs[i].Since = at
|
|
}
|
|
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
|
Resources: rs}, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service",
|
|
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
|
|
account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator",
|
|
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"}
|
|
say(unit)
|
|
say(unit) // lights not working
|
|
say(account, unit)
|
|
say(account, unit) // now it waits for a login
|
|
var resolved []string
|
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; {
|
|
time.Sleep(20 * time.Millisecond)
|
|
for _, e := range told.Said() {
|
|
if e.Change == conditions.ChangeCleared {
|
|
resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
|
|
// The clearing line is held to the plain rule too (ADR 0253).
|
|
w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation,
|
|
Resolved: e.Condition.Resolved, Needs: e.Condition.Needs}
|
|
if why, ok := conditions.PlainWords(w, "laptop"); !ok {
|
|
t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") {
|
|
t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
|
|
}
|
|
}
|
|
|
|
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
|
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
|
user := func(groups ...any) catalogue.Manifest {
|
|
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
|
"name": "operator", "groups": groups}}}
|
|
}
|
|
none := catalogue.Manifest{Module: "lights"}
|
|
for _, c := range []struct {
|
|
name string
|
|
from catalogue.Manifest
|
|
had bool
|
|
to catalogue.Manifest
|
|
addsSome bool
|
|
}{
|
|
{"a group added", none, true, user("lights"), true},
|
|
{"the same group kept", user("lights"), true, user("lights"), false},
|
|
{"a second group added", user("lights"), true, user("lights", "video"), true},
|
|
{"a group taken away", user("lights", "video"), true, user("lights"), false},
|
|
{"new to the machine, with a group", none, false, user("lights"), true},
|
|
{"new to the machine, no group", none, false, none, false},
|
|
} {
|
|
if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome {
|
|
t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome)
|
|
}
|
|
}
|
|
}
|