diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index e821b716..714dce2d 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -442,11 +442,12 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs words[node] = aboutTheMachine(node, moved, *g.Since, facts) } // **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its - // machines', its passes are counted apart, and the send's verdict is still one — but a module that was - // healthy on its own for the passes the gate asks keeps a pass when another beside it fails. + // machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send + // leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the + // settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until + // the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is + // put back with what failed, so none is left on the machine unjudged for other walks to wait on. worst, why := healthGood, "" - var failing []string - broken := map[string]bool{} reading := map[string]health{} waits := map[string]string{} var modules []string @@ -465,18 +466,21 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if _, seen := reading[j.module]; !seen { modules = append(modules, j.module) } + if h == healthBroken && !slices.Contains(g.Broken, j.module) { + g.Broken = append(g.Broken, j.module) + if g.BrokenWhy == "" { + g.BrokenWhy = said + } + } + if slices.Contains(g.Broken, j.module) { + h = healthBroken // found broken once, broken for the rest of the judging + } if h > reading[j.module] { reading[j.module] = h } if h == healthPerson { waits[j.module] = joinSaid(waits[j.module], said) } - if h > healthPerson && !slices.Contains(failing, j.module) { - failing = append(failing, j.module) - } - if h == healthBroken { - broken[j.module] = true - } if h > worst { worst, why = h, said } else if h == worst && h > healthPerson && why == "" { @@ -486,54 +490,77 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs if g.Healthy == nil { g.Healthy = map[string]int{} } + if g.HealthyAt == nil { + g.HealthyAt = map[string]time.Time{} + } g.Waits = nil + var failing []string for _, m := range modules { - switch { - case reading[m] > healthPerson: + if reading[m] > healthPerson { + failing = append(failing, m) g.Healthy[m] = 0 - default: + delete(g.HealthyAt, m) + continue + } + // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more + // often while another module beside it is not yet healthy counts no faster. + if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { g.Healthy[m]++ - if w := waits[m]; w != "" { - if g.Waits == nil { - g.Waits = map[string]string{} - } - g.Waits[m] = w + g.HealthyAt[m] = now + } + if w := waits[m]; w != "" { + if g.Waits == nil { + g.Waits = map[string]string{} } + g.Waits[m] = w } } - // passedAlone is every module that passed on its own while the send as a whole did not. - passedAlone := func() []string { - var out []string - if now.Sub(*g.Since) < gateSettle { - return nil - } + settled := now.Sub(*g.Since) >= gateSettle + passedAlone := func(m string) bool { + return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses + } + // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. + fail := func(why string) { + g.Passing, g.Failing = nil, nil for _, m := range modules { - if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses { - out = append(out, m) + if passedAlone(m) { + g.Passing = append(g.Passing, m) + } else { + g.Failing = append(g.Failing, m) } } - return out + decide(g, inventory.GateFailed, why, now) } + pastBound := now.Sub(*g.Since) > gateBound switch { case worst == healthBroken: - // What broke is put back; what was only not yet healthy beside it is too — they moved together. - g.Failing, g.Passing = failing, passedAlone() - decide(g, inventory.GateFailed, why, now) + var judging []string + for _, m := range modules { + if reading[m] != healthBroken && !passedAlone(m) { + judging = append(judging, m) + } + } + if len(judging) == 0 || pastBound { + fail(g.BrokenWhy) + break + } + // What broke fails the send; what is beside it is judged to its own verdict first, within the bound. + g.Passes, g.LastPass, g.Failing = 0, nil, failing + g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", ")) case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — // the provider's own condition says what is wrong (ADR 0240 rule 5). g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing case worst == healthNotYet: g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing - if now.Sub(*g.Since) > gateBound { - g.Passing = passedAlone() - decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now) + if pastBound { + fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) } default: // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil - if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle { + if g.Passes >= gatePasses && settled { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) } @@ -541,6 +568,18 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs return g.Verdict, nil } +// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait +// for a person, never another's (issue 318 review). +func whyFor(g *inventory.PlanGate, module string) string { + why, _, _ := strings.Cut(g.Why, waitsPrefix) + if w := g.Waits[module]; w != "" { + why += waitsPrefix + w + } + return why +} + +const waitsPrefix = "; and it waits for a person: " + // waitsSaid is the waits for a person a passing gate carries, as its verdict says them. func waitsSaid(waits map[string]string) string { if len(waits) == 0 { @@ -555,7 +594,7 @@ func waitsSaid(waits map[string]string) string { for _, m := range modules { said = append(said, waits[m]) } - return "; and it waits for a person: " + strings.Join(said, "; ") + return waitsPrefix + strings.Join(said, "; ") } func joinSaid(a, b string) string { @@ -579,7 +618,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str g := state.Gate err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, - Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since}) + Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since}) if err != nil && state.Build != "" { fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err) } diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go new file mode 100644 index 00000000..3229da1f --- /dev/null +++ b/cmd/mesh-controller/gate_followup_test.go @@ -0,0 +1,279 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a +// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own +// wait, and a provider waiting for a login says who waits on it. + +// withGateBounds sets the gate's bounds for one test. +func withGateBounds(t *testing.T, settle, every, bound time.Duration) { + t.Helper() + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + gateSettle, gateEvery, gateBound = settle, every, bound + t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) +} + +// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says +// of its modules' resources. +func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts { + return func() gateFacts { + now := time.Now() + at := now + return gateFacts{now: now, + reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, + engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, + served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}}, + health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}}, + } + } +} + +func healthyContainer(module string) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, + State: link.StateHealthy} +} + +// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node +// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, +// counted over the gate's spacing — before it says its own. The broken one alone is put back. +func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) + since := time.Now().Add(-time.Second) + base := factsOn(t, "laptop", since, healthyContainer("app")) + wasGather := gatherGateFacts + t.Cleanup(func() { gatherGateFacts = wasGather }) + gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { + f := base() + f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, + At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}} + return f, nil + } + g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} + pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}} + judgings := 0 + for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); { + if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil { + t.Fatal(err) + } + judgings++ + time.Sleep(30 * time.Millisecond) + } + if g.Verdict != inventory.GateFailed || judgings < gatePasses { + t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict, + judgings, g) + } + if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) { + t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing) + } + if !strings.Contains(g.Why, "witness") { + t.Errorf("the verdict does not say what broke: %q", g.Why) + } +} + +// broker_runtime is the node tools' module name, a core component a witness judges. +const broker_runtime = "node-tools" + +// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick +// while a module beside it is not yet healthy counts the healthy one once. +func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + withGateBounds(t, 0, time.Hour, time.Hour) + since := time.Now().Add(-time.Minute) + base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", + Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) + wasGather := gatherGateFacts + t.Cleanup(func() { gatherGateFacts = wasGather }) + gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil } + g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} + pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}} + now := time.Now() + for i := 0; i < 3; i++ { + if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil { + t.Fatal(err) + } + } + if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 { + t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy) + } +} + +// **A carried build's pass carries its own wait, never another's** (review (e)). +func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + since := time.Now().Add(-time.Minute) + g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed, + Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}), + Waits: map[string]string{"late": "relogin needed on laptop: late waits"}, + Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"}, + {Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}} + passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "") + app, _, _ := inv.GateOf(ctx, "build-app-c2") + late, _, _ := inv.GateOf(ctx, "build-late-c2") + if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed { + t.Errorf("app's pass: %+v; want it without late's wait", app) + } + if !strings.Contains(late.Why, "relogin needed on laptop") { + t.Errorf("late's pass: %+v; want its own wait", late) + } +} + +// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send +// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the +// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it. +func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + // `late` waits on anchor for a gate: c1 sent, c2 registered and built. + for _, b := range []inventory.Build{ + {ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late", + Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, + {ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late", + Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, + } { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit}) + b.Manifest = manifest + b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit, + Asked: b.Asked}); err != nil { + t.Fatal(err) + } + if b.Commit == "l1" { + if _, err := inv.Assign(ctx, "anchor", "late"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1", + "late": "l1"}); err != nil { + t.Fatal(err) + } + } + } + wasMoves := machineMoves + t.Cleanup(func() { machineMoves = wasMoves }) + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + gather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := gather(ctx, open, component) + f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ + healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late", + State: link.StateUnhealthy, Reason: "down"}}}} + return f, err + } + gateEvery, gateBound = 0, 400*time.Millisecond + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { + advancePlans(ctx, g.open) + if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone { + break + } + time.Sleep(30 * time.Millisecond) + } + p := g.plan(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note) + } + if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed || + !strings.Contains(v.Why, "on its own") { + t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) + } + if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed { + t.Fatal("late's build is not marked failed at its gate") + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" { + t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit, + current["late"].Commit) + } +} + +// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that +// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent. +func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + k := conditionsFrom + register(t, open, catalogue.Manifest{Module: "db", Version: "1", + Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) + register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) + for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { + if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", + Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { + t.Fatal(err) + } + at := h0 + say := func(machine string, rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator", + Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"} + unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service", + Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} + shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", + State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} + for look := 0; look < 2; look++ { + say("anchor", account, unit) + say("laptop", shop) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + var keys []string + var c conditions.Condition + for _, x := range list { + keys = append(keys, x.Key) + if x.Key == "module.db.anchor.relogin-needed" { + c = x + } + } + if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) { + t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys) + } + if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") { + t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 7596f4a6..b40c6818 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -152,14 +152,23 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } sort.Strings(modules) seen := map[string]bool{} + // became is, per module, the kind of condition this statement says of it: what a standing one of the + // other kind turned into. + became := map[string]string{} heldOn := map[string]string{} providers := map[catalogue.Chosen]bool{} for _, m := range modules { // **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the // operator, never urgent, cleared on the first statement that no longer says it. if said, waits := personWait(m, node, unhealthy[m]); waits { - o := reloginObservation(m, node, said, unhealthy[m]) + o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m]) + // **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its + // consumers are held under it, and its own condition is where they are said. + if hold != nil { + sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) + } seen[o.Key()] = true + became[m] = kindReloginNeeded if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen { continue } @@ -176,14 +185,10 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi providers[p] = true continue } - if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 { - o.Severity = conditions.Urgent - o.Said += "; " + waitingWords(waiters) - o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) - o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters)) - } + sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m})) } seen[o.Key()] = true + became[m] = kindModuleUnhealthy c, isOpen := standing[o.Key()] if streaks[m] < moduleUnhealthyAfter && !isOpen { continue // unconfirmed: one statement can be wrong; `node show` lists it @@ -207,7 +212,18 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi if on, held := heldOn[key]; held { why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on) } - if _, err := k.Clear(ctx, key, why); err != nil { + // **A condition that became the other kind** is not "working again" (issue 318 review): its clearing + // line says what it became. + resolved := "" + switch { + case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded: + why = fmt.Sprintf("%s on %s now waits only for a new login", module, node) + resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node) + case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy: + why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node) + resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module) + } + if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil { problems = append(problems, err.Error()) } } @@ -229,7 +245,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error { var raisedAt *conditions.Condition for i, c := range hold.open { - if c.Key == moduleUnhealthyKey(p.Module, p.Node) { + if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) { raisedAt = &hold.open[i] } } @@ -246,35 +262,97 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata return nil } o := moduleUnhealthyObservation(p.Module, p.Node, rs) - if waiters := hold.waitersOn(p); len(waiters) > 0 { - o.Severity = conditions.Urgent - o.Said += "; " + waitingWords(waiters) - o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) + if said, waits := personWait(p.Module, p.Node, rs); waits { + o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs) } + if o.Key() != raisedAt.Key { + return nil // its own statement says it next + } + sayWaitingOn(&o, hold.waitersOn(p)) _, err := k.Observe(ctx, o) return err } +// reloginKey is a module's relogin-needed condition on a machine. +func reloginKey(module, node string) string { + return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded) +} + +// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29). +func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string { + if inv == nil { + return "" + } + n, err := inv.NodeByName(ctx, node) + if err != nil { + return "" + } + return n.Account +} + // reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the // operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its // plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a -// person in again, and a restart of the module's service would start it in the same session. -func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation { +// person in again, and a restart of the module's service would start it in the same session. operator is +// the machine's operator account, when known: the words say "your account" only for it. +func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation { o := moduleUnhealthyObservation(module, node, rs) o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said - w := reloginWords(module, node) + accounts := waitingAccounts(module, rs) + yours := operator != "" && len(accounts) > 0 + for _, a := range accounts { + yours = yours && a == operator + } + w := reloginWords(module, node, yours) o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil return o } -// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253, -// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can. -func reloginWords(module, node string) words { +// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253, +// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says +// the account waiting is the operator's own on that machine; otherwise the words do not claim it is. +func reloginWords(module, node string, yours bool) words { + if yours { + return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node), + Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node), + Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+ + "cannot run until you log in again. Its update is in place and nothing was undone.", + conditions.Capital(module), module), + Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)} + } return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node), - Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node), - Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+ - "cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module), - Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)} + Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node), + Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+ + "so %s cannot run until it logs in again. Its update is in place and nothing was undone.", + conditions.Capital(module), node, module), + Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)} +} + +// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted. +func waitingAccounts(module string, rs []inventory.ResourceHealth) []string { + seen := map[string]bool{} + var out []string + for _, r := range rs { + if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy && + strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] { + seen[accountOf(r)] = true + out = append(out, accountOf(r)) + } + } + sort.Strings(out) + return out +} + +// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone +// waits on it, whether it is not working or waits for a new login. +func sayWaitingOn(o *conditions.Observation, waiters []string) { + if len(waiters) == 0 { + return + } + o.Severity = conditions.Urgent + o.Said += "; " + waitingWords(waiters) + o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) + o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters)) } // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module, @@ -352,18 +430,14 @@ const kindReloginNeeded = "relogin-needed" // not in its group or that could not be read — is not a wait, and the module is judged as before. A // resource still starting is not unhealthy and does not make a wait either. Pure. func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) { - waiting := map[string]bool{} - var accounts []string - for _, r := range rs { - if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy && - strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] { - waiting[accountOf(r)] = true - accounts = append(accounts, accountOf(r)) - } - } + accounts := waitingAccounts(module, rs) if len(accounts) == 0 { return "", false } + waiting := map[string]bool{} + for _, a := range accounts { + waiting[a] = true + } var units []string for _, r := range rs { if r.Module != module || r.State != link.StateUnhealthy { @@ -377,7 +451,6 @@ func personWait(module, machine string, rs []inventory.ResourceHealth) (string, return "", false } } - sort.Strings(accounts) said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+ "running session began before it was; log out of every session and in again, or reboot", machine, module, strings.Join(accounts, ", ")) @@ -415,6 +488,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module) } wait, waits := personWait(module, machine, h.Resources) + // **Only the build whose own send put the account into its wait is excused** (issue 318 review): an + // account already waiting before this send was not brought to it by this build, and a later build of + // the module sent while the login is still owed is judged as before. Two seconds of skew between the + // machine's clock and the controller's, as the witness allows. + if waits { + for _, r := range h.Resources { + if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy && + strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) { + waits = false + } + } + } for _, r := range h.Resources { if r.Module != module { continue diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index 4670fa0c..2393cc71 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -75,11 +75,24 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { now := time.Now() since := now.Add(-time.Minute) + account := relogin("lights", "operator") + account.Since = since.Add(10 * time.Second) // the send put it in its wait f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}} + Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}} if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") { t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why) } + // **Only the send that put the account in its wait is excused** (issue 318 review): an account waiting + // since before the send is judged as before, so a later build of the module is not passed on an old wait. + older := f.health["laptop"] + older.Resources = []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")} + older.Resources[0].Since = since.Add(-time.Hour) + f.health["laptop"] = older + if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet { + t.Fatalf("a wait from before the send reads %v %q; want not yet", h, why) + } + f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, + Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}} h := f.health["laptop"] h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights", State: link.StateUnhealthy, Reason: "down"}) @@ -197,7 +210,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { - o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{ + o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) plainExample(t, o, "openrazer waits for a new login on g14", "Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+ @@ -206,9 +219,64 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { if len(o.Actions) != 0 || o.Needs == "" { t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions) } - // The kind's own wording, for a condition raised without words, says the same. - if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs || - w.Headline != o.Headline || len(w.Actions) != 0 { + // **Not "your account" when it is not the operator's** (issue 318 review). + other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ + relogin("openrazer", "guest"), userUnit("openrazer", "guest")}) + plainExample(t, other, "openrazer waits for a new login on g14", + "Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+ + "put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+ + "until it logs in again. Its update is in place and nothing was undone.") + if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") { + t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation) + } + // The kind's own wording, for a condition raised without words, names the module whole, dots and all, + // and never claims the account is the operator's. + w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"}) + if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 || + strings.Contains(w.Explanation, "your account") { t.Errorf("the kind's wording: %+v", w) } } + +// When a module not working turns into one waiting for a new login, and back, the clearing line says what it +// became, never that it works again (issue 318 review). +func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + store := conditions.NewInMemory() + told := &conditions.Told{} + k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) + t.Cleanup(func() { k.Close(context.Background()) }) + at := h0 + say := func(rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, + Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", + Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} + account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", + Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} + say(unit) + say(unit) // lights not working + say(account, unit) + say(account, unit) // now it waits for a login + var resolved []string + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { + time.Sleep(20 * time.Millisecond) + for _, e := range told.Said() { + if e.Change == conditions.ChangeCleared { + resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) + } + } + } + if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { + t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 56803b12..75746ed8 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -184,11 +184,12 @@ var plainWordings = map[string]func(conditions.Observation) words{ Resolved: conditions.Capital(thing) + " works again"} }), kindReloginNeeded: worded(func(o conditions.Observation) words { + // A module's name may hold dots: it is the id without its machine. module := "" - if o.Scope == conditions.ScopeModule { - module = idPart(o, 0) + if o.Scope == conditions.ScopeModule && o.Machine != "" { + module = strings.TrimSuffix(o.ID, "."+o.Machine) } - return reloginWords(orModule(module), machineOr(o, "a machine")) + return reloginWords(orModule(module), machineOr(o, "a machine"), false) }), kindProviderFailing: worded(func(o conditions.Observation) words { thing, consumer := conditions.ThingWords(o), idPart(o, 2) diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index ef480603..29e76cea 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -319,7 +319,7 @@ func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent } seen[c.Build] = true if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, - Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why, + Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module), Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err) } @@ -375,10 +375,7 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent continue } seen[c.Build] = true - why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why) - if w := g.Waits[c.Module]; w != "" { - why += "; and it waits for a person: " + w - } + why := passedAloneWhy(g, c.Module) if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why, Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { @@ -393,6 +390,15 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent return kept } +// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait. +func passedAloneWhy(g *inventory.PlanGate, module string) string { + why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why) + if w := g.Waits[module]; w != "" { + why += waitsPrefix + w + } + return why +} + // sentTheBuild is every machine running a module that was last sent this build of it. func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) { running, err := open.inventory.Running(ctx, module) diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 58c1fa0c..7d179181 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -703,7 +703,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if lead.Gate.Verdict != inventory.GatePassed { continue } - passedWith(state, state.GatedBy, lead.Gate) + passedWith(m, state, state.GatedBy, lead.Gate) } switch { case step.failed != "": @@ -937,14 +937,14 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate // on the first of them passed, and its pass was kept for every build it carried (passCarried). -func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) { +func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) { if s.Gate == nil { s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since} } if s.Gate.Verdict != "" { return } - s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took + s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took s.Gate.Passes, s.Gate.Kept = g.Passes, true } @@ -963,6 +963,18 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module g := state.Gate if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) { gateFailed(batched, open, p, module, state, machines, why) + } else if slices.Contains(g.Passing, module) && state.Build != "" { + // **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes + // asked while another of its send failed, its build has a verdict, and other walks do not wait on it. + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, + Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed, + Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil { + fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err) + } + state.Why = "passed on its own; stopped with the send that carried it: " + g.Why + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept", + strings.Join(g.Machines, ", "), p.Tier, g.Why, module) } else { state.Why = "not found wanting; stopped with the send that carried it: " + g.Why p.State = inventory.PlanFailed diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index dc93fbbc..a6e6b5fe 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -255,8 +255,13 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { t.Fatalf("%s's build was marked failed for the node tools' condition", m) } } - if !strings.Contains(p.Note, "app1 was not found wanting") { - t.Fatalf("the plan blames the module its gate was kept on: %s", p.Note) + // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 + // review): it is not blamed, and not left without a verdict. + if !strings.Contains(p.Note, "app1 passed on its own and is kept") { + t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) + } + if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app1's own pass was not kept: %+v", v) } } diff --git a/internal/conditions/store.go b/internal/conditions/store.go index bdee8e69..cf016f4a 100644 --- a/internal/conditions/store.go +++ b/internal/conditions/store.go @@ -280,6 +280,13 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error) // Clear removes a condition an observation says is resolved, and says so. False when none was open. func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) { + return k.ClearSaying(ctx, key, why, "") +} + +// ClearSaying is Clear with the line the operator reads when it clears, in place of the condition's own +// resolved line: for a condition that ends because it became another (novox/hq ADR 0254: a module not +// working that now waits for a new login is not "working again"). Empty keeps the condition's own. +func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bool, error) { for i := 0; i < tries; i++ { entry, found, err := k.store.Get(ctx, key) if err != nil { @@ -296,6 +303,9 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) { if err := k.stamp(&c); err != nil { return false, err } + if resolved != "" { + c.Resolved = resolved + } if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) { continue } else if err != nil { diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 70b9d253..b3176a65 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -152,6 +152,13 @@ type PlanGate struct { // Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its // pass, and is not put back. Passing []string `json:"passing,omitempty"` + // HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery + // after the one before (issue 318 review). + HealthyAt map[string]time.Time `json:"healthy_at,omitempty"` + // Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the + // first reason: the send fails, and the modules beside them are judged to their own verdict first. + Broken []string `json:"broken,omitempty"` + BrokenWhy string `json:"broken_why,omitempty"` } // CarriedMove is one module's build moving on a machine with a gated send.