package main import ( "context" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254, // issue 318). func relogin(module, account string) inventory.ResourceHealth { return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account, Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module + ", and its running session began before it was; log out of every session and in again, or reboot"} } func userUnit(module, account string) inventory.ResourceHealth { return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit, Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} } // Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as // before, so that a fault is never excused. func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { notInGroup := relogin("lights", "operator") notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" systemUnit := userUnit("lights", "operator") systemUnit.Account = "" otherAccount := userUnit("lights", "guest") container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights", State: link.StateUnhealthy, Reason: "down"} otherModule := userUnit("sound", "operator") starting := userUnit("lights", "operator") starting.State = link.StateStarting byTarget := relogin("lights", "operator") byTarget.Account = "" // an account said by its target alone is still that account for _, c := range []struct { name string rs []inventory.ResourceHealth waits bool }{ {"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true}, {"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}, true}, {"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true}, {"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true}, {"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true}, {"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false}, {"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false}, {"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false}, {"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false}, {"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false}, } { said, waits := personWait("lights", "laptop", c.rs) if waits != c.waits { t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits) continue } if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") { t.Errorf("%s: said %q", c.name, said) } } } // The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module // with a container down beside it is not yet healthy, as before. func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { now := time.Now() since := now.Add(-time.Minute) account := relogin("lights", "operator") f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}, groupsAdded: map[string]bool{"lights": true}} if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") { t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why) } // **Only a move that put the account in a new group is excused** (issue 318 review): a later build that // adds no group did not bring the wait, and is judged as before. f.groupsAdded["lights"] = false if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet { t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why) } f.groupsAdded["lights"] = true h := f.health["laptop"] h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights", State: link.StateUnhealthy, Reason: "down"}) f.health["laptop"] = h if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet { t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why) } } // The module's condition says the wait in one sentence for a person, and clears on the first statement that // no longer says it — said as the module's own fault when its unit still fails after the new login. func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { open := aMesh(t) ctx := t.Context() k, _ := withConditionsInMemory(t) at := h0 say := func(rs ...link.ResourceHealth) { t.Helper() at = at.Add(time.Second) for i := range rs { rs[i].Since = at } if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { t.Fatal(err) } } asLink := func(r inventory.ResourceHealth) link.ResourceHealth { return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Account: r.Account} } account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) openNow := func() []conditions.Condition { t.Helper() list, err := k.Open(ctx) if err != nil { t.Fatal(err) } return list } say(account, unit) say(account, unit) list := openNow() if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { t.Fatalf("raised %+v; want lights' relogin-needed alone", list) } c := list[0] if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) } // After the new login the account is healthy, and the unit, still failing, is the module's own fault. account.State, account.Reason = link.StateHealthy, "" say(account, unit) list = openNow() if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) } unit.State, unit.Reason = link.StateHealthy, "" say(account, unit) if list = openNow(); len(list) != 0 { t.Fatalf("after the unit runs: %+v; want nothing open", list) } } // A module that is healthy on its own keeps its pass when another module of the same send fails at the // bound: it is neither put back nor left without a verdict for every other walk to wait on. func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { b := aBacklog(t) ctx := t.Context() inv := b.open.inventory releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil } backlogFacts := gatherGateFacts gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { f, err := backlogFacts(ctx, open, component) f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}}}} return f, err } gateEvery, gateBound = 0, 300*time.Millisecond deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { advancePlans(ctx, b.open) if p := b.release(t); p.State != inventory.PlanRolling { break } time.Sleep(20 * time.Millisecond) } p := b.release(t) if p.State != inventory.PlanFailed { t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) } if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") { t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) } if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { t.Fatal("late's build is not marked failed at its gate") } current, _ := inv.CurrentBuilds(ctx) if current["app"].Commit != "c2" || current["late"].Commit != "c1" { t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, current["late"].Commit) } if !strings.Contains(p.Note, "passed on their own and kept: app") { t.Errorf("the walk does not say what kept its pass: %s", p.Note) } } // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) plainExample(t, o, "openrazer waits for a new login on g14", "Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+ "group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+ "update is in place and nothing was undone.") if len(o.Actions) != 0 || o.Needs == "" { t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions) } // **Not "your account" when it is not the operator's** (issue 318 review). other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ relogin("openrazer", "guest"), userUnit("openrazer", "guest")}) plainExample(t, other, "openrazer waits for a new login on g14", "Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+ "put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+ "until it logs in again. Its update is in place and nothing was undone.") if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") { t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation) } // The kind's own wording, for a condition raised without words, names the module whole, dots and all, // and never claims the account is the operator's. w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"}) if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 || strings.Contains(w.Explanation, "your account") { t.Errorf("the kind's wording: %+v", w) } } // When a module not working turns into one waiting for a new login, and back, the clearing line says what it // became, never that it works again (issue 318 review). func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { open := aMesh(t) ctx := t.Context() store := conditions.NewInMemory() told := &conditions.Told{} k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) t.Cleanup(func() { k.Close(context.Background()) }) at := h0 say := func(rs ...link.ResourceHealth) { t.Helper() at = at.Add(time.Second) for i := range rs { rs[i].Since = at } if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { t.Fatal(err) } } unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} say(unit) say(unit) // lights not working say(account, unit) say(account, unit) // now it waits for a login var resolved []string for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { time.Sleep(20 * time.Millisecond) for _, e := range told.Said() { if e.Change == conditions.ChangeCleared { resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) // The clearing line is held to the plain rule too (ADR 0253). w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} if why, ok := conditions.PlainWords(w, "laptop"); !ok { t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) } } } } if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) } } // Which moves add an account group, read from the builds' manifests (issue 318 review). func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { user := func(groups ...any) catalogue.Manifest { return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", "name": "operator", "groups": groups}}} } none := catalogue.Manifest{Module: "lights"} for _, c := range []struct { name string from catalogue.Manifest had bool to catalogue.Manifest addsSome bool }{ {"a group added", none, true, user("lights"), true}, {"the same group kept", user("lights"), true, user("lights"), false}, {"a second group added", user("lights"), true, user("lights", "video"), true}, {"a group taken away", user("lights", "video"), true, user("lights"), false}, {"new to the machine, with a group", none, false, user("lights"), true}, {"new to the machine, no group", none, false, none, false}, } { if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome { t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome) } } }