Compare commits

...
4 Commits
Author SHA1 Message Date
jochen bd25807ca7 Say a pending new login without 'session', which the plain-words rule refuses (hq ADR 0253, issue 318 review) 2026-10-08 15:06:30 +02:00
jochen c71af17421 Give every module of a failed send a verdict, and excuse only the wait a build's own send brought (hq issue 318 review) 2026-10-08 15:06:05 +02:00
mesh-admin 45ae1d0112 Merge pull request 'Offer only acknowledgements, keep a refused verdict, say a change of words (hq ADR 0258, issue 324)' (#145) from fix/notifications-after-review into main 2026-10-08 13:02:29 +00:00
jochen 76cfbac7a1 Offer only acknowledgements as answers, keep a refused verdict, and say a change of words
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/notifications-after-review merged: every member is merged
Review found that a desk click proves nothing about who chose, that refused words
could turn "Needs you" into "Nothing for you to do", that sound words were refused,
and that a quiet warning whose words came to need the operator was never said
(hq ADR 0258).
2026-10-08 14:53:36 +02:00
16 changed files with 834 additions and 162 deletions
+75 -36
View File
@@ -442,11 +442,12 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
}
// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
// machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send
// leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the
// settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until
// the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is
// put back with what failed, so none is left on the machine unjudged for other walks to wait on.
worst, why := healthGood, ""
var failing []string
broken := map[string]bool{}
reading := map[string]health{}
waits := map[string]string{}
var modules []string
@@ -465,18 +466,21 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
if _, seen := reading[j.module]; !seen {
modules = append(modules, j.module)
}
if h == healthBroken && !slices.Contains(g.Broken, j.module) {
g.Broken = append(g.Broken, j.module)
if g.BrokenWhy == "" {
g.BrokenWhy = said
}
}
if slices.Contains(g.Broken, j.module) {
h = healthBroken // found broken once, broken for the rest of the judging
}
if h > reading[j.module] {
reading[j.module] = h
}
if h == healthPerson {
waits[j.module] = joinSaid(waits[j.module], said)
}
if h > healthPerson && !slices.Contains(failing, j.module) {
failing = append(failing, j.module)
}
if h == healthBroken {
broken[j.module] = true
}
if h > worst {
worst, why = h, said
} else if h == worst && h > healthPerson && why == "" {
@@ -486,54 +490,77 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
if g.Healthy == nil {
g.Healthy = map[string]int{}
}
if g.HealthyAt == nil {
g.HealthyAt = map[string]time.Time{}
}
g.Waits = nil
var failing []string
for _, m := range modules {
switch {
case reading[m] > healthPerson:
if reading[m] > healthPerson {
failing = append(failing, m)
g.Healthy[m] = 0
default:
delete(g.HealthyAt, m)
continue
}
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
// often while another module beside it is not yet healthy counts no faster.
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
g.Healthy[m]++
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
g.HealthyAt[m] = now
}
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
}
}
// passedAlone is every module that passed on its own while the send as a whole did not.
passedAlone := func() []string {
var out []string
if now.Sub(*g.Since) < gateSettle {
return nil
}
settled := now.Sub(*g.Since) >= gateSettle
passedAlone := func(m string) bool {
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
}
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
fail := func(why string) {
g.Passing, g.Failing = nil, nil
for _, m := range modules {
if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
out = append(out, m)
if passedAlone(m) {
g.Passing = append(g.Passing, m)
} else {
g.Failing = append(g.Failing, m)
}
}
return out
decide(g, inventory.GateFailed, why, now)
}
pastBound := now.Sub(*g.Since) > gateBound
switch {
case worst == healthBroken:
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
g.Failing, g.Passing = failing, passedAlone()
decide(g, inventory.GateFailed, why, now)
var judging []string
for _, m := range modules {
if reading[m] != healthBroken && !passedAlone(m) {
judging = append(judging, m)
}
}
if len(judging) == 0 || pastBound {
fail(g.BrokenWhy)
break
}
// What broke fails the send; what is beside it is judged to its own verdict first, within the bound.
g.Passes, g.LastPass, g.Failing = 0, nil, failing
g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", "))
case worst == healthWaiting:
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
// the provider's own condition says what is wrong (ADR 0240 rule 5).
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
case worst == healthNotYet:
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
if now.Sub(*g.Since) > gateBound {
g.Passing = passedAlone()
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
if pastBound {
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
}
default:
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
if g.Passes >= gatePasses && settled {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
}
@@ -541,6 +568,18 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
return g.Verdict, nil
}
// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
// for a person, never another's (issue 318 review).
func whyFor(g *inventory.PlanGate, module string) string {
why, _, _ := strings.Cut(g.Why, waitsPrefix)
if w := g.Waits[module]; w != "" {
why += waitsPrefix + w
}
return why
}
const waitsPrefix = "; and it waits for a person: "
// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
func waitsSaid(waits map[string]string) string {
if len(waits) == 0 {
@@ -555,7 +594,7 @@ func waitsSaid(waits map[string]string) string {
for _, m := range modules {
said = append(said, waits[m])
}
return "; and it waits for a person: " + strings.Join(said, "; ")
return waitsPrefix + strings.Join(said, "; ")
}
func joinSaid(a, b string) string {
@@ -579,7 +618,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str
g := state.Gate
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since})
if err != nil && state.Build != "" {
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
}
+279
View File
@@ -0,0 +1,279 @@
package main
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
// wait, and a provider waiting for a login says who waits on it.
// withGateBounds sets the gate's bounds for one test.
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
t.Helper()
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery, gateBound = settle, every, bound
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
}
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
// of its modules' resources.
func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
return func() gateFacts {
now := time.Now()
at := now
return gateFacts{now: now,
reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{},
served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
}
}
}
func healthyContainer(module string) inventory.ResourceHealth {
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
State: link.StateHealthy}
}
// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
since := time.Now().Add(-time.Second)
base := factsOn(t, "laptop", since, healthyContainer("app"))
wasGather := gatherGateFacts
t.Cleanup(func() { gatherGateFacts = wasGather })
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
f := base()
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
return f, nil
}
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
judgings := 0
for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
t.Fatal(err)
}
judgings++
time.Sleep(30 * time.Millisecond)
}
if g.Verdict != inventory.GateFailed || judgings < gatePasses {
t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
judgings, g)
}
if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
}
if !strings.Contains(g.Why, "witness") {
t.Errorf("the verdict does not say what broke: %q", g.Why)
}
}
// broker_runtime is the node tools' module name, a core component a witness judges.
const broker_runtime = "node-tools"
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
// while a module beside it is not yet healthy counts the healthy one once.
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, 0, time.Hour, time.Hour)
since := time.Now().Add(-time.Minute)
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
wasGather := gatherGateFacts
t.Cleanup(func() { gatherGateFacts = wasGather })
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
now := time.Now()
for i := 0; i < 3; i++ {
if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
t.Fatal(err)
}
}
if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
}
}
// **A carried build's pass carries its own wait, never another's** (review (e)).
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
since := time.Now().Add(-time.Minute)
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
app, _, _ := inv.GateOf(ctx, "build-app-c2")
late, _, _ := inv.GateOf(ctx, "build-late-c2")
if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
t.Errorf("app's pass: %+v; want it without late's wait", app)
}
if !strings.Contains(late.Why, "relogin needed on laptop") {
t.Errorf("late's pass: %+v; want its own wait", late)
}
}
// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
for _, b := range []inventory.Build{
{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
b.Manifest = manifest
b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
Asked: b.Asked}); err != nil {
t.Fatal(err)
}
if b.Commit == "l1" {
if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
"late": "l1"}); err != nil {
t.Fatal(err)
}
}
}
wasMoves := machineMoves
t.Cleanup(func() { machineMoves = wasMoves })
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
gather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f, err := gather(ctx, open, component)
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
State: link.StateUnhealthy, Reason: "down"}}}}
return f, err
}
gateEvery, gateBound = 0, 400*time.Millisecond
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
advancePlans(ctx, g.open)
if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
break
}
time.Sleep(30 * time.Millisecond)
}
p := g.plan(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
}
if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
!strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
}
if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
t.Fatal("late's build is not marked failed at its gate")
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
current["late"].Commit)
}
}
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
k := conditionsFrom
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
t.Fatal(err)
}
at := h0
say := func(machine string, rs ...link.ResourceHealth) {
t.Helper()
at = at.Add(time.Second)
for i := range rs {
rs[i].Since = at
}
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
t.Fatal(err)
}
}
account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
for look := 0; look < 2; look++ {
say("anchor", account, unit)
say("laptop", shop)
}
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
var keys []string
var c conditions.Condition
for _, x := range list {
keys = append(keys, x.Key)
if x.Key == "module.db.anchor.relogin-needed" {
c = x
}
}
if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
}
if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
}
}
+5 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
@@ -65,8 +66,10 @@ var handActVerbs = []handActVerb{
// not trusted with it — either is a repair the owner should have made.
{Verb: "plans go"},
{Verb: "broker consumer-reset"},
// Silencing the same condition twice says the condition, or what it watches, wants mending.
{Verb: "conditions silence"},
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
{Verb: "conditions silence", Decision: "the operator's answer on a notification is their decision, " +
"not a repair (ADR 0258)", DecidedFor: []string{conditions.CauseOperatorAnswer}},
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
// cause since, so no act recorded through it now carries it.
+125 -37
View File
@@ -152,14 +152,23 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
sort.Strings(modules)
seen := map[string]bool{}
// became is, per module, the kind of condition this statement says of it: what a standing one of the
// other kind turned into.
became := map[string]string{}
heldOn := map[string]string{}
providers := map[catalogue.Chosen]bool{}
for _, m := range modules {
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
// operator, never urgent, cleared on the first statement that no longer says it.
if said, waits := personWait(m, node, unhealthy[m]); waits {
o := reloginObservation(m, node, said, unhealthy[m])
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
// consumers are held under it, and its own condition is where they are said.
if hold != nil {
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
}
seen[o.Key()] = true
became[m] = kindReloginNeeded
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
continue
}
@@ -176,14 +185,10 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
providers[p] = true
continue
}
if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
}
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
}
seen[o.Key()] = true
became[m] = kindModuleUnhealthy
c, isOpen := standing[o.Key()]
if streaks[m] < moduleUnhealthyAfter && !isOpen {
continue // unconfirmed: one statement can be wrong; `node show` lists it
@@ -207,7 +212,18 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
if on, held := heldOn[key]; held {
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
}
if _, err := k.Clear(ctx, key, why); err != nil {
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
// line says what it became.
resolved := ""
switch {
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
}
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
problems = append(problems, err.Error())
}
}
@@ -229,7 +245,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
var raisedAt *conditions.Condition
for i, c := range hold.open {
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
raisedAt = &hold.open[i]
}
}
@@ -246,42 +262,104 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
return nil
}
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
if waiters := hold.waitersOn(p); len(waiters) > 0 {
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
if said, waits := personWait(p.Module, p.Node, rs); waits {
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
}
if o.Key() != raisedAt.Key {
return nil // its own statement says it next
}
sayWaitingOn(&o, hold.waitersOn(p))
_, err := k.Observe(ctx, o)
return err
}
// reloginKey is a module's relogin-needed condition on a machine.
func reloginKey(module, node string) string {
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
}
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
if inv == nil {
return ""
}
n, err := inv.NodeByName(ctx, node)
if err != nil {
return ""
}
return n.Account
}
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
// person in again, and a restart of the module's service would start it in the same session.
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
// person in again, and a restart of the module's service would start it in the same session. operator is
// the machine's operator account, when known: the words say "your account" only for it.
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
o := moduleUnhealthyObservation(module, node, rs)
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
w := reloginWords(module, node)
accounts := waitingAccounts(module, rs)
yours := operator != "" && len(accounts) > 0
for _, a := range accounts {
yours = yours && a == operator
}
w := reloginWords(module, node, yours)
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
return o
}
// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
func reloginWords(module, node string) words {
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
func reloginWords(module, node string, yours bool) words {
if yours {
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
"cannot run until you log in again. Its update is in place and nothing was undone.",
conditions.Capital(module), module),
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
}
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
conditions.Capital(module), node, module),
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
}
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
seen := map[string]bool{}
var out []string
for _, r := range rs {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
seen[accountOf(r)] = true
out = append(out, accountOf(r))
}
}
sort.Strings(out)
return out
}
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
// waits on it, whether it is not working or waits for a new login.
func sayWaitingOn(o *conditions.Observation, waiters []string) {
if len(waiters) == 0 {
return
}
o.Severity = conditions.Urgent
o.Said += "; " + waitingWords(waiters)
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
}
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
var words, said, plain []string
needs, actions := moduleNeeds(node, rs)
needs := moduleNeeds(node, rs)
for _, r := range rs {
plain = append(plain, resourcePlainWords(r))
if r.Kind == link.KindUnit {
@@ -305,12 +383,15 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
namesWords(plain, 3)),
Needs: needs,
Actions: actions,
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
}
// reasonWords is why a resource is unhealthy, as a person reads it.
func reasonWords(r inventory.ResourceHealth) string {
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
}
switch r.Reason {
case "restarting":
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
@@ -353,18 +434,14 @@ const kindReloginNeeded = "relogin-needed"
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
// resource still starting is not unhealthy and does not make a wait either. Pure.
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
waiting := map[string]bool{}
var accounts []string
for _, r := range rs {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
waiting[accountOf(r)] = true
accounts = append(accounts, accountOf(r))
}
}
accounts := waitingAccounts(module, rs)
if len(accounts) == 0 {
return "", false
}
waiting := map[string]bool{}
for _, a := range accounts {
waiting[a] = true
}
var units []string
for _, r := range rs {
if r.Module != module || r.State != link.StateUnhealthy {
@@ -378,13 +455,12 @@ func personWait(module, machine string, rs []inventory.ResourceHealth) (string,
return "", false
}
}
sort.Strings(accounts)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
"running session began before it was; log out of every session and in again, or reboot", machine, module,
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
strings.Join(accounts, ", "))
if len(units) > 0 {
sort.Strings(units)
said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
}
return said, true
}
@@ -416,6 +492,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
}
wait, waits := personWait(module, machine, h.Resources)
// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
// account already waiting before this send was not brought to it by this build, and a later build of
// the module sent while the login is still owed is judged as before. Two seconds of skew between the
// machine's clock and the controller's, as the witness allows.
if waits {
for _, r := range h.Resources {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
waits = false
}
}
}
for _, r := range h.Resources {
if r.Module != module {
continue
+73 -5
View File
@@ -75,11 +75,24 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
now := time.Now()
since := now.Add(-time.Minute)
account := relogin("lights", "operator")
account.Since = since.Add(10 * time.Second) // the send put it in its wait
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}}
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
}
// **Only the send that put the account in its wait is excused** (issue 318 review): an account waiting
// since before the send is judged as before, so a later build of the module is not passed on an old wait.
older := f.health["laptop"]
older.Resources = []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}
older.Resources[0].Since = since.Add(-time.Hour)
f.health["laptop"] = older
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
t.Fatalf("a wait from before the send reads %v %q; want not yet", h, why)
}
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}
h := f.health["laptop"]
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
@@ -197,7 +210,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
// no button, since nothing but their own new login can do it.
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
plainExample(t, o, "openrazer waits for a new login on g14",
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
@@ -206,9 +219,64 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
if len(o.Actions) != 0 || o.Needs == "" {
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
}
// The kind's own wording, for a condition raised without words, says the same.
if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs ||
w.Headline != o.Headline || len(w.Actions) != 0 {
// **Not "your account" when it is not the operator's** (issue 318 review).
other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
relogin("openrazer", "guest"), userUnit("openrazer", "guest")})
plainExample(t, other, "openrazer waits for a new login on g14",
"Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+
"put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+
"until it logs in again. Its update is in place and nothing was undone.")
if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") {
t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation)
}
// The kind's own wording, for a condition raised without words, names the module whole, dots and all,
// and never claims the account is the operator's.
w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"})
if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 ||
strings.Contains(w.Explanation, "your account") {
t.Errorf("the kind's wording: %+v", w)
}
}
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
// became, never that it works again (issue 318 review).
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
store := conditions.NewInMemory()
told := &conditions.Told{}
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told})
t.Cleanup(func() { k.Close(context.Background()) })
at := h0
say := func(rs ...link.ResourceHealth) {
t.Helper()
at = at.Add(time.Second)
for i := range rs {
rs[i].Since = at
}
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
Resources: rs}, at); err != nil {
t.Fatal(err)
}
}
unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service",
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator",
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"}
say(unit)
say(unit) // lights not working
say(account, unit)
say(account, unit) // now it waits for a login
var resolved []string
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; {
time.Sleep(20 * time.Millisecond)
for _, e := range told.Said() {
if e.Change == conditions.ChangeCleared {
resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
}
}
}
if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") {
t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
}
}
+20 -37
View File
@@ -184,11 +184,12 @@ var plainWordings = map[string]func(conditions.Observation) words{
Resolved: conditions.Capital(thing) + " works again"}
}),
kindReloginNeeded: worded(func(o conditions.Observation) words {
// A module's name may hold dots: it is the id without its machine.
module := ""
if o.Scope == conditions.ScopeModule {
module = idPart(o, 0)
if o.Scope == conditions.ScopeModule && o.Machine != "" {
module = strings.TrimSuffix(o.ID, "."+o.Machine)
}
return reloginWords(orModule(module), machineOr(o, "a machine"))
return reloginWords(orModule(module), machineOr(o, "a machine"), false)
}),
kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
@@ -642,43 +643,28 @@ func waitingNeeds(severity conditions.Severity) string {
return ""
}
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
// the module that should have said go is the one not answering.
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
if severity != conditions.Urgent {
return nil
}
return []conditions.Action{
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
}
}
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
var actions []conditions.Action
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
unit := ""
for _, r := range rs {
if strings.Contains(r.Reason, "relogin needed") {
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
return reloginNeeds(node)
}
if r.Kind == link.KindUnit && len(actions) < 2 {
scope := "system"
if strings.Contains(r.Reason, "account's own") {
scope = "user"
}
label := "Restart"
if len(actions) > 0 {
label = "Restart " + unitPlainWords(r.Target)
}
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
if r.Kind == link.KindUnit && unit == "" {
unit = unitPlainWords(r.Target)
}
}
if len(actions) > 0 {
return "restart it; if it fails again, the details say why.", actions
if unit != "" {
return fmt.Sprintf("restart its service %s on %s; if it fails again, the details say why.", unit, node)
}
return "", nil
return ""
}
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
func reloginNeeds(node string) string {
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
}
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
@@ -695,18 +681,15 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
long = "for " + humanDuration(d)
}
if o.Resolver == conditions.ResolverOperator {
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
// performs it (ADR 0258).
switch held {
case "held":
needs = "release it, or stop it."
actions = []conditions.Action{
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
}
case "ready", "checked":
needs = "merge its pull request, or close it."
default:
needs = "stop it, or read the details to see what it waits for."
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
}
}
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
+45 -22
View File
@@ -70,13 +70,7 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
plainExample(t, got[0], "openrazer delivery waiting to start",
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
"be stuck.", "Start", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
t.Errorf("start: %+v", a)
}
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
t.Errorf("stop: %+v", a)
}
"be stuck.")
// Many modules are counted, not listed in the headline.
f.waits[0].modules = []string{"a", "b", "c", "d"}
@@ -87,24 +81,35 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
}
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
// account's own service manager (exit-code)". The operator restarts it from the notification.
func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words
// and offered as no answer (ADR 0258).
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
plainExample(t, o, "openrazer not working on g14",
"Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
t.Errorf("restart: %+v", a)
}
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
"Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. "+
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
"as it runs again.")
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
if o.Needs != "log out of g14 completely and log in again, or restart it." || len(o.Actions) != 0 {
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
}
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
if why, ok := conditions.PlainWords(w, "g14"); !ok {
t.Errorf("the relogin words are not plain: %s", why)
}
// And the kind issue 318 raises for it (ADR 0254).
rw := plainWordings["relogin-needed"](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14",
Machine: "g14", Kind: "relogin-needed", Severity: conditions.Warning})
if why, ok := conditions.PlainWords(rw, "g14"); !ok || rw.Needs == "" || len(rw.Actions) != 0 {
t.Errorf("relogin-needed: %s %+v", why, rw)
}
if strings.Contains(o.Summary, "session") || !strings.Contains(o.Summary, "waits for a new login of jochen") {
t.Errorf("relogin summary: %q", o.Summary)
}
}
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
@@ -143,15 +148,12 @@ func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
plainExample(t, got[0], "Delivery of hq held for 36 hours",
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
"Release", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
t.Errorf("release: %+v", a)
}
)
}
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
@@ -184,3 +186,24 @@ func TestEveryWordingIsPlain(t *testing.T) {
}
}
}
// **An answer on a notification is no repair** (novox/hq ADR 0258): silencing chosen by the operator on the
// desk does not count toward a healer wanted; the same silence by hand for another cause still does.
func TestAnOperatorsAnswerIsNoHandRepair(t *testing.T) {
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
f := calm(now)
for i := 0; i < 3; i++ {
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), conditions.CauseOperatorAnswer)
a.Verb = "conditions silence"
f.handActs = append(f.handActs, a)
}
if got := watchHandActs(f); len(got) != 0 {
t.Fatalf("an answer counted as a repair: %+v", got)
}
for i := range f.handActs {
f.handActs[i].Cause = "machine-units"
}
if got := watchHandActs(f); len(got) != 1 {
t.Fatalf("a silence by hand stopped counting: %+v", got)
}
}
+11 -5
View File
@@ -319,7 +319,7 @@ func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
}
seen[c.Build] = true
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module),
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
}
@@ -375,10 +375,7 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
continue
}
seen[c.Build] = true
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why)
if w := g.Waits[c.Module]; w != "" {
why += "; and it waits for a person: " + w
}
why := passedAloneWhy(g, c.Module)
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
@@ -393,6 +390,15 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
return kept
}
// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
func passedAloneWhy(g *inventory.PlanGate, module string) string {
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
if w := g.Waits[module]; w != "" {
why += waitsPrefix + w
}
return why
}
// sentTheBuild is every machine running a module that was last sent this build of it.
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
running, err := open.inventory.Running(ctx, module)
+15 -3
View File
@@ -703,7 +703,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
if lead.Gate.Verdict != inventory.GatePassed {
continue
}
passedWith(state, state.GatedBy, lead.Gate)
passedWith(m, state, state.GatedBy, lead.Gate)
}
switch {
case step.failed != "":
@@ -937,14 +937,14 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
if s.Gate == nil {
s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since}
}
if s.Gate.Verdict != "" {
return
}
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took
s.Gate.Passes, s.Gate.Kept = g.Passes, true
}
@@ -963,6 +963,18 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
g := state.Gate
if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
gateFailed(batched, open, p, module, state, machines, why)
} else if slices.Contains(g.Passing, module) && state.Build != "" {
// **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes
// asked while another of its send failed, its build has a verdict, and other walks do not wait on it.
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed,
Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil {
fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err)
}
state.Why = "passed on its own; stopped with the send that carried it: " + g.Why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept",
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
} else {
state.Why = "not found wanting; stopped with the send that carried it: " + g.Why
p.State = inventory.PlanFailed
-1
View File
@@ -361,7 +361,6 @@ func watchWaits(f *signalFacts) []conditions.Observation {
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: walkWaitingWords(w, in, severity),
Needs: waitingNeeds(severity),
Actions: waitingActions(w, severity),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
}
return out
+7 -2
View File
@@ -255,8 +255,13 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
t.Fatalf("%s's build was marked failed for the node tools' condition", m)
}
}
if !strings.Contains(p.Note, "app1 was not found wanting") {
t.Fatalf("the plan blames the module its gate was kept on: %s", p.Note)
// Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318
// review): it is not blamed, and not left without a verdict.
if !strings.Contains(p.Note, "app1 passed on its own and is kept") {
t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note)
}
if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed {
t.Fatalf("app1's own pass was not kept: %+v", v)
}
}
+3
View File
@@ -36,6 +36,9 @@ const (
ChangeSilenced = "silenced"
ChangeUnsilenced = "silence-ended"
ChangeCleared = "cleared"
// ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR
// 0258): a warning kept quiet as needing nothing is said once its words need the operator.
ChangeWords = "words"
)
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
+69 -10
View File
@@ -62,10 +62,37 @@ const (
)
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
// answer to a condition the operator decided to live with.
// answer to a condition the operator decided to live with. It is an acknowledgement (ADR 0234 §8): the
// only kind of answer a desk click performs until answers are authorised (novox/hq ADR 0258). Its cause
// marks it as an answer, which the hand-act log does not count as a repair.
func SilenceAction(key string) Action {
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
Arguments: map[string]string{"silence": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
}
// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with.
const CauseOperatorAnswer = "operator-answer"
// FallbackNeeds is what a condition needs when its producer's words needed the operator and could not
// be said: the verdict is kept, never turned into "Nothing for you to do." (ADR 0258).
const FallbackNeeds = "read the details to see what to do."
func sameActions(a, b []Action) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].Label != b[i].Label || a[i].Verb != b[i].Verb || a[i].Machine != b[i].Machine ||
len(a[i].Arguments) != len(b[i].Arguments) {
return false
}
for k, v := range a[i].Arguments {
if b[i].Arguments[k] != v {
return false
}
}
}
return true
}
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
@@ -108,7 +135,9 @@ var (
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b`)
byHand = regexp.MustCompile(`(?i)\bby hand\b`)
latinAbbr = regexp.MustCompile(`\b(e\.g|i\.e|etc)\.`)
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
)
@@ -126,14 +155,15 @@ func Plain(text string, machines ...string) (string, bool) {
if m := markup.FindString(text); m != "" {
return "markup (" + m + ")", false
}
text = latinAbbr.ReplaceAllString(text, "eg")
for _, shape := range []struct {
re *regexp.Regexp
what string
}{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"},
{clockTime, "a clock time or date"}, {flag, "a command's flag"}} {
if m := shape.re.FindString(text); m != "" {
if shape.re == hexID && !strings.ContainsAny(m, "0123456789") {
continue // a word of letters a to f only
if shape.re == hexID && (!strings.ContainsAny(m, "0123456789") || !strings.ContainsAny(m, "abcdef")) {
continue // letters a to f only, or a number: words and counts, not a hash
}
return shape.what + " (" + strings.TrimSpace(m) + ")", false
}
@@ -158,16 +188,14 @@ func PlainWords(w Words, machines ...string) (string, bool) {
if len(w.Explanation) > ExplanationMax {
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
}
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
return "explanation: sends the operator elsewhere (" + m + ")", false
}
if m := agentWord.FindString(w.Explanation); m != "" {
return "explanation: sends the operator elsewhere (" + m + ")", false
}
if w.Needs != "" {
if why, ok := Plain(w.Needs, machines...); !ok {
return "needs: " + why, false
}
if m := agentWord.FindString(w.Needs); m != "" {
if m := agentWord.FindString(w.Needs) + byHand.FindString(w.Needs); m != "" {
return "needs: not something the operator does themselves (" + m + ")", false
}
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
@@ -221,6 +249,8 @@ func plainly(o Observation) Observation {
if w.Headline != "" && w.Resolved == "" {
w.Resolved = "Resolved: " + lowerFirst(w.Headline)
}
// An explanation too long is cut, never refused: what it says first is what matters.
w.Explanation = cut(w.Explanation, ExplanationMax-len(Verdict(w.Needs, ""))-1)
if from == "" {
if Unworded != nil {
Unworded(o, "the kind "+o.Kind+" has no plain words")
@@ -230,13 +260,42 @@ func plainly(o Observation) Observation {
if Unworded != nil {
Unworded(o, from+" is not plain: "+why)
}
// The scope's words, but never a weaker verdict: what needed the operator still does, and its
// answers are kept where they are themselves sound.
needed, actions := w.Needs != "", soundActions(w.Actions)
w = scopeWords(o)
if needed {
w.Needs, w.Actions = FallbackNeeds, actions
}
}
o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
o.Explanation = Verdict(w.Needs, w.Explanation)
return o
}
// soundActions are the actions with a label and a verb.
func soundActions(list []Action) []Action {
var out []Action
for _, a := range list {
if a.Label != "" && len(a.Label) <= 24 && a.Verb != "" {
out = append(out, a)
}
}
return out
}
// cut shortens text to at most n bytes at a word, ending it with an ellipsis.
func cut(text string, n int) string {
if len(text) <= n || n < 10 {
return text
}
t := text[:n-3]
if i := strings.LastIndexByte(t, ' '); i > n/2 {
t = t[:i]
}
return strings.TrimRight(t, " ,;:") + "…"
}
// Verdict is an explanation opened by its verdict.
func Verdict(needs, explanation string) string {
if needs == "" {
+86 -2
View File
@@ -11,7 +11,7 @@ import (
func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) {
refused := map[string]string{
"the walk of novox/mesh-catalog a6385479 waits": "a hash",
"plan plan-1791454185265004861 is waiting": "a hash",
"plan plan-1791454185265004861 is waiting": "a numbered id",
"see `mesh-delivery.show` for why": "markup",
"call mesh-delivery.show for why": "a dotted name",
"key plan.x.waiting": "a dotted name",
@@ -124,7 +124,6 @@ func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
for _, w := range []Words{
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
} {
@@ -161,3 +160,88 @@ func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
t.Errorf("seen again after escalation: %q", c.Explanation)
}
}
// **What the plain rule refused wrongly** (novox/hq ADR 0258): a login session is the operator's own word,
// "e.g." is not a dotted name, a long count is not a hash, and a long explanation is cut, not refused.
func TestThePlainRuleRefusesNoPlainWords(t *testing.T) {
for _, text := range []string{
"log out of every session on g14 and log in again.",
"A module, e.g. the lighting daemon, waits.",
"It holds 12345678 files.",
"A person repaired push by hand 35 times in 14 days.",
} {
if why, ok := Plain(text, "g14"); !ok {
t.Errorf("%q refused: %s", text, why)
}
}
if _, ok := Plain("commit 0123abc landed"); ok {
t.Error("a hash passed")
}
k, _, _, _ := keeper(t)
long := strings.Repeat("The bus refused a message. ", 30)
Wording("test-long", func(o Observation) Words {
return Words{Headline: "A long one", Explanation: long, Resolved: "Done"}
})
before := Unworded
var unworded []string
Unworded = func(o Observation, why string) { unworded = append(unworded, why) }
t.Cleanup(func() { Unworded = before })
c, err := k.Observe(t.Context(), Observation{Scope: ScopeBus, ID: "x", Kind: "test-long", Severity: Warning,
Source: "test", Summary: "long"})
if err != nil {
t.Fatal(err)
}
if c.Headline != "A long one" || len(c.Explanation) > ExplanationMax || !strings.HasSuffix(c.Explanation, "…") ||
len(unworded) != 0 {
t.Errorf("a long explanation was not cut: %d %q %v", len(c.Explanation), c.Headline, unworded)
}
}
// **Words refused keep their verdict and their answers** (ADR 0258): a producer that needed the operator
// still does, said as "read the details", and its sound answers stay.
func TestRefusedWordsKeepTheirVerdictAndAnswers(t *testing.T) {
k, _, _, _ := keeper(t)
before := Unworded
Unworded = func(Observation, string) {}
t.Cleanup(func() { Unworded = before })
silence := SilenceAction("module.openrazer.g14.x")
c, err := k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-refused",
Machine: "g14", Severity: Warning, Source: "test", Summary: "s",
Headline: "openrazer at 0123abc failed", Needs: "restart it.", Explanation: "It failed.",
Actions: []Action{silence}})
if err != nil {
t.Fatal(err)
}
if c.Needs != FallbackNeeds || !strings.HasPrefix(c.Explanation, NeedsYou+" "+FallbackNeeds) ||
len(c.Actions) != 1 || c.Actions[0].Label != silence.Label || strings.Contains(c.Headline, "0123abc") {
t.Errorf("the verdict or the answers were lost: %+v", c)
}
// Words that needed nothing stay "Nothing for you to do.".
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "y", Kind: "test-refused",
Machine: "g14", Severity: Warning, Source: "test", Summary: "s", Headline: "at 0123abc", Explanation: "x."})
if c.Needs != "" || !strings.HasPrefix(c.Explanation, NothingToDo) {
t.Errorf("%+v", c)
}
}
// **Words that now need the operator are said as a change** (ADR 0258): a condition whose needs or answers
// change emits condition-changed, so a warning kept quiet is said.
func TestAChangeOfWordsIsSaid(t *testing.T) {
k, _, told, _ := keeper(t)
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
if _, err := k.Observe(t.Context(), o); err != nil {
t.Fatal(err)
}
if _, err := k.Observe(t.Context(), o); err != nil { // the same words: no event
t.Fatal(err)
}
o.Needs = "check that ace is on."
if _, err := k.Observe(t.Context(), o); err != nil {
t.Fatal(err)
}
said := settled(t, told, 2)
if len(said) != 2 || said[1].Change != ChangeWords || said[1].Event != EventChanged || said[1].Needs != o.Needs {
t.Fatalf("%+v", said)
}
}
+14
View File
@@ -240,8 +240,12 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
// The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
wasNeeds, wasActions := c.Needs, c.Actions
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
escalatedWords(&c)
if c.Needs != wasNeeds || !sameActions(c.Actions, wasActions) {
changes = append(changes, Event{Change: ChangeWords, Was: wasNeeds})
}
if o.Machine != "" {
c.Subject.Machine = o.Machine
}
@@ -276,6 +280,13 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
return k.ClearSaying(ctx, key, why, "")
}
// ClearSaying is Clear with the line the operator reads when it clears, in place of the condition's own
// resolved line: for a condition that ends because it became another (novox/hq ADR 0254: a module not
// working that now waits for a new login is not "working again"). Empty keeps the condition's own.
func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bool, error) {
for i := 0; i < tries; i++ {
entry, found, err := k.store.Get(ctx, key)
if err != nil {
@@ -292,6 +303,9 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
if err := k.stamp(&c); err != nil {
return false, err
}
if resolved != "" {
c.Resolved = resolved
}
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
continue
} else if err != nil {
+7
View File
@@ -152,6 +152,13 @@ type PlanGate struct {
// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
// pass, and is not put back.
Passing []string `json:"passing,omitempty"`
// HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery
// after the one before (issue 318 review).
HealthyAt map[string]time.Time `json:"healthy_at,omitempty"`
// Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the
// first reason: the send fails, and the modules beside them are judged to their own verdict first.
Broken []string `json:"broken,omitempty"`
BrokenWhy string `json:"broken_why,omitempty"`
}
// CarriedMove is one module's build moving on a machine with a gated send.