Compare commits

...
4 Commits
Author SHA1 Message Date
jochen 76cfbac7a1 Offer only acknowledgements as answers, keep a refused verdict, and say a change of words
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/notifications-after-review delivered: every member is delivered
Review found that a desk click proves nothing about who chose, that refused words
could turn "Needs you" into "Nothing for you to do", that sound words were refused,
and that a quiet warning whose words came to need the operator was never said
(hq ADR 0258).
2026-10-08 14:53:36 +02:00
mesh-admin 28712eaea1 Merge pull request 'Pass a wait for a person's new login, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318)' (#142) from fix/318-a-wait-for-a-person-is-not-a-failure into main 2026-10-08 12:41:08 +00:00
jochen 51d7bbfdc8 Give relogin-needed its plain words: it needs the operator and offers no button, since only their new login can end it (hq ADR 0253, ADR 0254)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/318-a-wait-for-a-person-is-not-a-failure delivered: every member is delivered
2026-10-08 14:25:20 +02:00
jochen e3b5c224e8 Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318) 2026-10-08 14:20:55 +02:00
16 changed files with 967 additions and 80 deletions
+93 -6
View File
@@ -77,6 +77,10 @@ type health int
const (
healthGood health = iota
// healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for
// one person's new login (personWait). The build did what it should; no bound a machine can meet ends
// the wait. It counts as a pass, and the gate carries the reading along in its verdict.
healthPerson
// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
// build for something it did not do.
@@ -209,7 +213,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
firstLine(f.openErr.Error())
}
for _, c := range f.open {
if c.Source == gateProbe || c.Raised.Before(since) {
// A wait for a person's new login is the module's reading, not a fault raised since the send: the
// gate reads it from the statement below (ADR 0254).
if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded {
continue
}
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
@@ -435,13 +441,19 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
for node, moved := range byMachine {
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
}
// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
worst, why := healthGood, ""
var failing []string
broken := map[string]bool{}
reading := map[string]health{}
waits := map[string]string{}
var modules []string
for _, j := range pairs {
w := words[j.node]
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
if h == healthGood {
if h == healthGood || h == healthPerson {
if on, named := w.on[j.module]; named {
h, said = healthNotYet, on
} else if w.whole != "" {
@@ -450,7 +462,16 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
h, said = healthWaiting, w.waiting
}
}
if h != healthGood && !slices.Contains(failing, j.module) {
if _, seen := reading[j.module]; !seen {
modules = append(modules, j.module)
}
if h > reading[j.module] {
reading[j.module] = h
}
if h == healthPerson {
waits[j.module] = joinSaid(waits[j.module], said)
}
if h > healthPerson && !slices.Contains(failing, j.module) {
failing = append(failing, j.module)
}
if h == healthBroken {
@@ -458,14 +479,45 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
}
if h > worst {
worst, why = h, said
} else if h == worst && h != healthGood && why == "" {
} else if h == worst && h > healthPerson && why == "" {
why = said
}
}
if g.Healthy == nil {
g.Healthy = map[string]int{}
}
g.Waits = nil
for _, m := range modules {
switch {
case reading[m] > healthPerson:
g.Healthy[m] = 0
default:
g.Healthy[m]++
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
}
}
}
// passedAlone is every module that passed on its own while the send as a whole did not.
passedAlone := func() []string {
var out []string
if now.Sub(*g.Since) < gateSettle {
return nil
}
for _, m := range modules {
if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
out = append(out, m)
}
}
return out
}
switch {
case worst == healthBroken:
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
g.Failing = failing
g.Failing, g.Passing = failing, passedAlone()
decide(g, inventory.GateFailed, why, now)
case worst == healthWaiting:
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
@@ -474,19 +526,48 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
case worst == healthNotYet:
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
if now.Sub(*g.Since) > gateBound {
g.Passing = passedAlone()
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
}
default:
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second)), now)
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
}
}
return g.Verdict, nil
}
// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
func waitsSaid(waits map[string]string) string {
if len(waits) == 0 {
return ""
}
modules := make([]string, 0, len(waits))
for m := range waits {
modules = append(modules, m)
}
sort.Strings(modules)
var said []string
for _, m := range modules {
said = append(said, waits[m])
}
return "; and it waits for a person: " + strings.Join(said, "; ")
}
func joinSaid(a, b string) string {
switch {
case a == "":
return b
case b == "" || strings.Contains(a, b):
return a
}
return a + "; " + b
}
// decide sets a gate's verdict.
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
@@ -959,6 +1040,12 @@ func gateLine(g *inventory.PlanGate) string {
} else if g.Verdict == "" {
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
}
if g.Verdict == "" && len(g.Waits) > 0 {
line += waitsSaid(g.Waits)
}
if len(g.Passing) > 0 {
line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ")
}
if g.Rollback != "" {
line += "; " + g.Rollback
}
+5 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
@@ -65,8 +66,10 @@ var handActVerbs = []handActVerb{
// not trusted with it — either is a repair the owner should have made.
{Verb: "plans go"},
{Verb: "broker consumer-reset"},
// Silencing the same condition twice says the condition, or what it watches, wants mending.
{Verb: "conditions silence"},
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
{Verb: "conditions silence", Decision: "the operator's answer on a notification is their decision, " +
"not a repair (ADR 0258)", DecidedFor: []string{conditions.CauseOperatorAnswer}},
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
// cause since, so no act recorded through it now carries it.
+115 -4
View File
@@ -73,7 +73,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
for _, r := range h.Resources {
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
Check: r.Check, Needs: r.Needs}
Check: r.Check, Needs: r.Needs, Account: r.Account}
resources = append(resources, kept)
if r.State == link.StateUnhealthy && r.Module != "" {
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
@@ -135,7 +135,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
standing := map[string]conditions.Condition{}
for _, c := range open {
if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded) && c.Subject.Machine == node {
standing[c.Key] = c
}
}
@@ -155,6 +155,19 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
heldOn := map[string]string{}
providers := map[catalogue.Chosen]bool{}
for _, m := range modules {
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
// operator, never urgent, cleared on the first statement that no longer says it.
if said, waits := personWait(m, node, unhealthy[m]); waits {
o := reloginObservation(m, node, said, unhealthy[m])
seen[o.Key()] = true
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
continue
}
if _, err := k.Observe(ctx, o); err != nil {
problems = append(problems, err.Error())
}
continue
}
o := moduleUnhealthyObservation(m, node, unhealthy[m])
if hold != nil {
if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
@@ -188,6 +201,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
}
module := strings.TrimSuffix(c.Subject.ID, "."+node)
why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
if c.Kind == kindReloginNeeded {
why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
}
if on, held := heldOn[key]; held {
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
}
@@ -239,11 +255,33 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
return err
}
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
// person in again, and a restart of the module's service would start it in the same session.
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
o := moduleUnhealthyObservation(module, node, rs)
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
w := reloginWords(module, node)
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
return o
}
// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
func reloginWords(module, node string) words {
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
}
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
var words, said, plain []string
needs, actions := moduleNeeds(node, rs)
needs := moduleNeeds(node, rs)
for _, r := range rs {
plain = append(plain, resourcePlainWords(r))
if r.Kind == link.KindUnit {
@@ -267,7 +305,6 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
namesWords(plain, 3)),
Needs: needs,
Actions: actions,
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
}
@@ -296,9 +333,76 @@ func orNotSaid(s string) string {
return s
}
// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
const kindReloginNeeded = "relogin-needed"
// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
// excused:
//
// - at least one of the module's resources is its account (kind account), unhealthy with a reason that
// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
// running began before;
// - every other unhealthy resource of the module runs in the own service manager of one of those very
// accounts (Account names it): the manager that began before the group and does not hold it.
//
// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
// another account's manager, a resource whose engine does not say whose manager it is in, an account
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
// resource still starting is not unhealthy and does not make a wait either. Pure.
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
waiting := map[string]bool{}
var accounts []string
for _, r := range rs {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
waiting[accountOf(r)] = true
accounts = append(accounts, accountOf(r))
}
}
if len(accounts) == 0 {
return "", false
}
var units []string
for _, r := range rs {
if r.Module != module || r.State != link.StateUnhealthy {
continue
}
switch {
case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
units = append(units, r.Target)
default:
return "", false
}
}
sort.Strings(accounts)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
"running session began before it was; log out of every session and in again, or reboot", machine, module,
strings.Join(accounts, ", "))
if len(units) > 0 {
sort.Strings(units)
said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
}
return said, true
}
// accountOf is the account a resource of kind account is: named by the engine, or its target.
func accountOf(r inventory.ResourceHealth) string {
if r.Account != "" {
return r.Account
}
return r.Target
}
// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
//
// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
// and not something a machine can meet within a bound.
func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
if f.healthErr != nil {
return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
@@ -310,10 +414,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
if h.HeardAt.Before(since) {
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
}
wait, waits := personWait(module, machine, h.Resources)
for _, r := range h.Resources {
if r.Module != module {
continue
}
if waits && r.State == link.StateUnhealthy {
continue
}
switch r.State {
case link.StateHealthy:
case link.StateStarting:
@@ -329,6 +437,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
reasonAfter(r.Reason))
}
}
if waits {
return healthPerson, wait
}
return healthGood, ""
}
+214
View File
@@ -0,0 +1,214 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254,
// issue 318).
func relogin(module, account string) inventory.ResourceHealth {
return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account,
Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module +
", and its running session began before it was; log out of every session and in again, or reboot"}
}
func userUnit(module, account string) inventory.ResourceHealth {
return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit,
Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
Reason: "failed in the account's own service manager (exit-code)"}
}
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
// before, so that a fault is never excused.
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
notInGroup := relogin("lights", "operator")
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
systemUnit := userUnit("lights", "operator")
systemUnit.Account = ""
otherAccount := userUnit("lights", "guest")
container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
State: link.StateUnhealthy, Reason: "down"}
otherModule := userUnit("sound", "operator")
starting := userUnit("lights", "operator")
starting.State = link.StateStarting
byTarget := relogin("lights", "operator")
byTarget.Account = "" // an account said by its target alone is still that account
for _, c := range []struct {
name string
rs []inventory.ResourceHealth
waits bool
}{
{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
userUnit("lights", "operator")}, true},
{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
} {
said, waits := personWait("lights", "laptop", c.rs)
if waits != c.waits {
t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
continue
}
if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
t.Errorf("%s: said %q", c.name, said)
}
}
}
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
// with a container down beside it is not yet healthy, as before.
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
now := time.Now()
since := now.Add(-time.Minute)
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
}
h := f.health["laptop"]
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
f.health["laptop"] = h
if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet {
t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why)
}
}
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
// no longer says it — said as the module's own fault when its unit still fails after the new login.
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
at := h0
say := func(rs ...link.ResourceHealth) {
t.Helper()
at = at.Add(time.Second)
for i := range rs {
rs[i].Since = at
}
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
Resources: rs}, at); err != nil {
t.Fatal(err)
}
}
asLink := func(r inventory.ResourceHealth) link.ResourceHealth {
return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State,
Reason: r.Reason, Account: r.Account}
}
account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator"))
openNow := func() []conditions.Condition {
t.Helper()
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
return list
}
say(account, unit)
say(account, unit)
list := openNow()
if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" {
t.Fatalf("raised %+v; want lights' relogin-needed alone", list)
}
c := list[0]
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator ||
!strings.HasPrefix(c.Summary, "relogin needed on laptop:") {
t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary)
}
// After the new login the account is healthy, and the unit, still failing, is the module's own fault.
account.State, account.Reason = link.StateHealthy, ""
say(account, unit)
list = openNow()
if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" {
t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list)
}
unit.State, unit.Reason = link.StateHealthy, ""
say(account, unit)
if list = openNow(); len(list) != 0 {
t.Fatalf("after the unit runs: %+v; want nothing open", list)
}
}
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"laptop": true}, nil
}
backlogFacts := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f, err := backlogFacts(ctx, open, component)
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
Resources: []inventory.ResourceHealth{
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy,
Reason: "down"}}}}
return f, err
}
gateEvery, gateBound = 0, 300*time.Millisecond
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
advancePlans(ctx, b.open)
if p := b.release(t); p.State != inventory.PlanRolling {
break
}
time.Sleep(20 * time.Millisecond)
}
p := b.release(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note)
}
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
!strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
}
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed {
t.Fatal("late's build is not marked failed at its gate")
}
current, _ := inv.CurrentBuilds(ctx)
if current["app"].Commit != "c2" || current["late"].Commit != "c1" {
t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit,
current["late"].Commit)
}
if !strings.Contains(p.Note, "passed on their own and kept: app") {
t.Errorf("the walk does not say what kept its pass: %s", p.Note)
}
}
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
// no button, since nothing but their own new login can do it.
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
plainExample(t, o, "openrazer waits for a new login on g14",
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
"group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+
"update is in place and nothing was undone.")
if len(o.Actions) != 0 || o.Needs == "" {
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
}
// The kind's own wording, for a condition raised without words, says the same.
if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs ||
w.Headline != o.Headline || len(w.Actions) != 0 {
t.Errorf("the kind's wording: %+v", w)
}
}
+31 -34
View File
@@ -183,6 +183,13 @@ var plainWordings = map[string]func(conditions.Observation) words{
Explanation: fmt.Sprintf("%s is not healthy. It clears as soon as it runs again.", conditions.Capital(thing)),
Resolved: conditions.Capital(thing) + " works again"}
}),
kindReloginNeeded: worded(func(o conditions.Observation) words {
module := ""
if o.Scope == conditions.ScopeModule {
module = idPart(o, 0)
}
return reloginWords(orModule(module), machineOr(o, "a machine"))
}),
kindProviderFailing: worded(func(o conditions.Observation) words {
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
if consumer == "" {
@@ -635,43 +642,28 @@ func waitingNeeds(severity conditions.Severity) string {
return ""
}
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
// the module that should have said go is the one not answering.
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
if severity != conditions.Urgent {
return nil
}
return []conditions.Action{
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
}
}
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
var actions []conditions.Action
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
unit := ""
for _, r := range rs {
if strings.Contains(r.Reason, "relogin needed") {
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
return reloginNeeds(node)
}
if r.Kind == link.KindUnit && len(actions) < 2 {
scope := "system"
if strings.Contains(r.Reason, "account's own") {
scope = "user"
}
label := "Restart"
if len(actions) > 0 {
label = "Restart " + unitPlainWords(r.Target)
}
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
if r.Kind == link.KindUnit && unit == "" {
unit = unitPlainWords(r.Target)
}
}
if len(actions) > 0 {
return "restart it; if it fails again, the details say why.", actions
if unit != "" {
return fmt.Sprintf("restart its service %s on %s; if it fails again, the details say why.", unit, node)
}
return "", nil
return ""
}
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
func reloginNeeds(node string) string {
return fmt.Sprintf("log out of every session on %s and log in again.", node)
}
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
@@ -688,18 +680,15 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
long = "for " + humanDuration(d)
}
if o.Resolver == conditions.ResolverOperator {
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
// performs it (ADR 0258).
switch held {
case "held":
needs = "release it, or stop it."
actions = []conditions.Action{
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
}
case "ready", "checked":
needs = "merge its pull request, or close it."
default:
needs = "stop it, or read the details to see what it waits for."
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
}
}
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
@@ -759,3 +748,11 @@ func unitReasonWords(reason string) string {
}
return "failed"
}
// orModule is a module's name, or "a module" when none is known.
func orModule(m string) string {
if m == "" {
return "a module"
}
return m
}
+41 -21
View File
@@ -70,13 +70,7 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
plainExample(t, got[0], "openrazer delivery waiting to start",
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
"be stuck.", "Start", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
t.Errorf("start: %+v", a)
}
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
t.Errorf("stop: %+v", a)
}
"be stuck.")
// Many modules are counted, not listed in the headline.
f.waits[0].modules = []string{"a", "b", "c", "d"}
@@ -87,24 +81,32 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
}
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
// account's own service manager (exit-code)". The operator restarts it from the notification.
func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words
// and offered as no answer (ADR 0258).
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
plainExample(t, o, "openrazer not working on g14",
"Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
t.Errorf("restart: %+v", a)
}
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
"Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. "+
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
"as it runs again.")
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
}
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
if why, ok := conditions.PlainWords(w, "g14"); !ok {
t.Errorf("the relogin words are not plain: %s", why)
}
// And the kind issue 318 raises for it (ADR 0254).
rw := plainWordings["relogin-needed"](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14",
Machine: "g14", Kind: "relogin-needed", Severity: conditions.Warning})
if why, ok := conditions.PlainWords(rw, "g14"); !ok || rw.Needs == "" || len(rw.Actions) != 0 {
t.Errorf("relogin-needed: %s %+v", why, rw)
}
}
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
@@ -143,15 +145,12 @@ func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
plainExample(t, got[0], "Delivery of hq held for 36 hours",
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
"Release", "Stop")
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
t.Errorf("release: %+v", a)
}
)
}
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
@@ -184,3 +183,24 @@ func TestEveryWordingIsPlain(t *testing.T) {
}
}
}
// **An answer on a notification is no repair** (novox/hq ADR 0258): silencing chosen by the operator on the
// desk does not count toward a healer wanted; the same silence by hand for another cause still does.
func TestAnOperatorsAnswerIsNoHandRepair(t *testing.T) {
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
f := calm(now)
for i := 0; i < 3; i++ {
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), conditions.CauseOperatorAnswer)
a.Verb = "conditions silence"
f.handActs = append(f.handActs, a)
}
if got := watchHandActs(f); len(got) != 0 {
t.Fatalf("an answer counted as a repair: %+v", got)
}
for i := range f.handActs {
f.handActs[i].Cause = "machine-units"
}
if got := watchHandActs(f); len(got) != 1 {
t.Fatalf("a silence by hand stopped counting: %+v", got)
}
}
+37
View File
@@ -334,6 +334,15 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
notes = append(notes, p.Note)
}
done := map[string]bool{except: true}
// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
// left without a verdict is one more move every other walk would wait for.
if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 {
notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", "))
for _, m := range kept {
done[m] = true
}
}
for _, c := range g.Carried {
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
continue
@@ -356,6 +365,34 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
p.Note = strings.Join(notes, "; ")
}
// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the
// send failed (ADR 0254), except the plan's own module; answers the modules kept.
func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string {
var kept []string
seen := map[string]bool{}
for _, c := range g.Carried {
if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) {
continue
}
seen[c.Build] = true
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why)
if w := g.Waits[c.Module]; w != "" {
why += "; and it waits for a person: " + w
}
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module,
c.Node, err)
continue
}
if !slices.Contains(kept, c.Module) {
kept = append(kept, c.Module)
}
}
return kept
}
// sentTheBuild is every machine running a module that was last sent this build of it.
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
running, err := open.inventory.Running(ctx, module)
+244
View File
@@ -0,0 +1,244 @@
package main
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the
// older commit: the statements are read from the node-engine's JSON, as the controller hears them.
//
// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send —
// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged
// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the
// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service
// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its
// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and
// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there.
//
// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along,
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
// for the operator, that a new login is needed on the laptop.
func TestReplay318(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
keeper, _ := withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
build := func(module, repository, commit string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
type move struct{ module, repository, from, to string }
moves := []move{
{"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"},
{"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"},
{"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"},
}
machines := []string{"laptop", "workstation"}
if _, err := inv.AddNode(ctx, "workstation"); err != nil {
t.Fatal(err)
}
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
for _, mv := range moves {
build(mv.module, mv.repository, mv.from, old)
for _, n := range machines {
if _, err := inv.Assign(ctx, n, mv.module); err != nil {
t.Fatal(err)
}
}
}
for _, n := range machines {
sent := map[string]string{}
for _, mv := range moves {
sent[mv.module] = mv.from
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
t.Fatal(err)
}
}
for _, mv := range moves {
build(mv.module, mv.repository, mv.to, recent)
}
// What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the
// daemon's unit failed in that account's own manager; docker's container is healthy.
statement := func(at time.Time) link.Health {
raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [
{"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service",
"state": "healthy", "since": %q},
{"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator",
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
"reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"},
{"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service",
"account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
"reason": "failed in the account's own service manager (exit-code)"}]}`,
link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano),
at.Format(time.RFC3339Nano))
var h link.Health
if err := json.Unmarshal([]byte(raw), &h); err != nil {
t.Fatal(err)
}
return h
}
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
})
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"laptop": true, "workstation": true}, nil
}
var sends [][]string
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
sends = append(sends, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
modules, _ := open.inventory.Assigned(ctx, node)
carried := map[string]string{}
for _, m := range modules {
carried[m] = current[m].Commit
}
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
// Every judging hears each machine's newest statement first, as the controller does between judgings:
// the account's wait and the failed unit are raised as openrazer's condition, after the send.
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
now := time.Now()
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{},
judged: true}
for _, m := range machines {
if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil {
return f, err
}
f.served[m] = served{runtime: true, tools: map[string]bool{}}
}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
f.reports[r.Node] = r
}
if f.health, err = open.inventory.Healths(ctx); err != nil {
return f, err
}
f.open, f.openErr = keeper.Open(ctx)
return f, nil
}
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
gateSettle, gateEvery, gateBound = 0, 0, 0
for i := 0; i < 12; i++ {
advancePlans(ctx, open)
time.Sleep(5 * time.Millisecond)
}
var p inventory.Plan
plans, err := inv.RecentPlans(ctx, 10)
if err != nil {
t.Fatal(err)
}
for _, q := range plans {
if q.Release != nil {
p = q
break
}
}
if p.Release == nil {
t.Fatal("no walk of the builds waiting for a gate was opened")
}
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) {
t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State,
p.Release.Done, p.Note)
}
if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) {
t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends)
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
t.Fatal(err)
}
if current["openrazer"].Commit != "c78b5fc9" {
t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit)
}
for _, mv := range moves {
v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to)
if err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err)
}
if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") {
t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why)
}
}
// What the operator reads: one condition per machine for openrazer, saying a new login is needed there,
// a warning for a person, and no fault of the build.
list, err := keeper.Open(ctx)
if err != nil {
t.Fatal(err)
}
var said []string
for _, c := range list {
said = append(said, c.Key+": "+c.Summary)
if c.Subject.ID == "openrazer.laptop" {
if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent ||
c.Resolver != conditions.ResolverOperator {
t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary)
}
}
}
if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") {
t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n"))
}
}
-1
View File
@@ -361,7 +361,6 @@ func watchWaits(f *signalFacts) []conditions.Observation {
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
Explanation: walkWaitingWords(w, in, severity),
Needs: waitingNeeds(severity),
Actions: waitingActions(w, severity),
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
}
return out
+3
View File
@@ -36,6 +36,9 @@ const (
ChangeSilenced = "silenced"
ChangeUnsilenced = "silence-ended"
ChangeCleared = "cleared"
// ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR
// 0258): a warning kept quiet as needing nothing is said once its words need the operator.
ChangeWords = "words"
)
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
+69 -10
View File
@@ -62,10 +62,37 @@ const (
)
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
// answer to a condition the operator decided to live with.
// answer to a condition the operator decided to live with. It is an acknowledgement (ADR 0234 §8): the
// only kind of answer a desk click performs until answers are authorised (novox/hq ADR 0258). Its cause
// marks it as an answer, which the hand-act log does not count as a repair.
func SilenceAction(key string) Action {
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
Arguments: map[string]string{"silence": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
}
// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with.
const CauseOperatorAnswer = "operator-answer"
// FallbackNeeds is what a condition needs when its producer's words needed the operator and could not
// be said: the verdict is kept, never turned into "Nothing for you to do." (ADR 0258).
const FallbackNeeds = "read the details to see what to do."
func sameActions(a, b []Action) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].Label != b[i].Label || a[i].Verb != b[i].Verb || a[i].Machine != b[i].Machine ||
len(a[i].Arguments) != len(b[i].Arguments) {
return false
}
for k, v := range a[i].Arguments {
if b[i].Arguments[k] != v {
return false
}
}
}
return true
}
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
@@ -108,7 +135,9 @@ var (
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
agentWord = regexp.MustCompile(`(?i)\bagents?\b`)
byHand = regexp.MustCompile(`(?i)\bby hand\b`)
latinAbbr = regexp.MustCompile(`\b(e\.g|i\.e|etc)\.`)
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
)
@@ -126,14 +155,15 @@ func Plain(text string, machines ...string) (string, bool) {
if m := markup.FindString(text); m != "" {
return "markup (" + m + ")", false
}
text = latinAbbr.ReplaceAllString(text, "eg")
for _, shape := range []struct {
re *regexp.Regexp
what string
}{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"},
{clockTime, "a clock time or date"}, {flag, "a command's flag"}} {
if m := shape.re.FindString(text); m != "" {
if shape.re == hexID && !strings.ContainsAny(m, "0123456789") {
continue // a word of letters a to f only
if shape.re == hexID && (!strings.ContainsAny(m, "0123456789") || !strings.ContainsAny(m, "abcdef")) {
continue // letters a to f only, or a number: words and counts, not a hash
}
return shape.what + " (" + strings.TrimSpace(m) + ")", false
}
@@ -158,16 +188,14 @@ func PlainWords(w Words, machines ...string) (string, bool) {
if len(w.Explanation) > ExplanationMax {
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
}
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
return "explanation: sends the operator elsewhere (" + m + ")", false
}
if m := agentWord.FindString(w.Explanation); m != "" {
return "explanation: sends the operator elsewhere (" + m + ")", false
}
if w.Needs != "" {
if why, ok := Plain(w.Needs, machines...); !ok {
return "needs: " + why, false
}
if m := agentWord.FindString(w.Needs); m != "" {
if m := agentWord.FindString(w.Needs) + byHand.FindString(w.Needs); m != "" {
return "needs: not something the operator does themselves (" + m + ")", false
}
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
@@ -221,6 +249,8 @@ func plainly(o Observation) Observation {
if w.Headline != "" && w.Resolved == "" {
w.Resolved = "Resolved: " + lowerFirst(w.Headline)
}
// An explanation too long is cut, never refused: what it says first is what matters.
w.Explanation = cut(w.Explanation, ExplanationMax-len(Verdict(w.Needs, ""))-1)
if from == "" {
if Unworded != nil {
Unworded(o, "the kind "+o.Kind+" has no plain words")
@@ -230,13 +260,42 @@ func plainly(o Observation) Observation {
if Unworded != nil {
Unworded(o, from+" is not plain: "+why)
}
// The scope's words, but never a weaker verdict: what needed the operator still does, and its
// answers are kept where they are themselves sound.
needed, actions := w.Needs != "", soundActions(w.Actions)
w = scopeWords(o)
if needed {
w.Needs, w.Actions = FallbackNeeds, actions
}
}
o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
o.Explanation = Verdict(w.Needs, w.Explanation)
return o
}
// soundActions are the actions with a label and a verb.
func soundActions(list []Action) []Action {
var out []Action
for _, a := range list {
if a.Label != "" && len(a.Label) <= 24 && a.Verb != "" {
out = append(out, a)
}
}
return out
}
// cut shortens text to at most n bytes at a word, ending it with an ellipsis.
func cut(text string, n int) string {
if len(text) <= n || n < 10 {
return text
}
t := text[:n-3]
if i := strings.LastIndexByte(t, ' '); i > n/2 {
t = t[:i]
}
return strings.TrimRight(t, " ,;:") + "…"
}
// Verdict is an explanation opened by its verdict.
func Verdict(needs, explanation string) string {
if needs == "" {
+86 -2
View File
@@ -11,7 +11,7 @@ import (
func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) {
refused := map[string]string{
"the walk of novox/mesh-catalog a6385479 waits": "a hash",
"plan plan-1791454185265004861 is waiting": "a hash",
"plan plan-1791454185265004861 is waiting": "a numbered id",
"see `mesh-delivery.show` for why": "markup",
"call mesh-delivery.show for why": "a dotted name",
"key plan.x.waiting": "a dotted name",
@@ -124,7 +124,6 @@ func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
for _, w := range []Words{
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
} {
@@ -161,3 +160,88 @@ func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
t.Errorf("seen again after escalation: %q", c.Explanation)
}
}
// **What the plain rule refused wrongly** (novox/hq ADR 0258): a login session is the operator's own word,
// "e.g." is not a dotted name, a long count is not a hash, and a long explanation is cut, not refused.
func TestThePlainRuleRefusesNoPlainWords(t *testing.T) {
for _, text := range []string{
"log out of every session on g14 and log in again.",
"A module, e.g. the lighting daemon, waits.",
"It holds 12345678 files.",
"A person repaired push by hand 35 times in 14 days.",
} {
if why, ok := Plain(text, "g14"); !ok {
t.Errorf("%q refused: %s", text, why)
}
}
if _, ok := Plain("commit 0123abc landed"); ok {
t.Error("a hash passed")
}
k, _, _, _ := keeper(t)
long := strings.Repeat("The bus refused a message. ", 30)
Wording("test-long", func(o Observation) Words {
return Words{Headline: "A long one", Explanation: long, Resolved: "Done"}
})
before := Unworded
var unworded []string
Unworded = func(o Observation, why string) { unworded = append(unworded, why) }
t.Cleanup(func() { Unworded = before })
c, err := k.Observe(t.Context(), Observation{Scope: ScopeBus, ID: "x", Kind: "test-long", Severity: Warning,
Source: "test", Summary: "long"})
if err != nil {
t.Fatal(err)
}
if c.Headline != "A long one" || len(c.Explanation) > ExplanationMax || !strings.HasSuffix(c.Explanation, "…") ||
len(unworded) != 0 {
t.Errorf("a long explanation was not cut: %d %q %v", len(c.Explanation), c.Headline, unworded)
}
}
// **Words refused keep their verdict and their answers** (ADR 0258): a producer that needed the operator
// still does, said as "read the details", and its sound answers stay.
func TestRefusedWordsKeepTheirVerdictAndAnswers(t *testing.T) {
k, _, _, _ := keeper(t)
before := Unworded
Unworded = func(Observation, string) {}
t.Cleanup(func() { Unworded = before })
silence := SilenceAction("module.openrazer.g14.x")
c, err := k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-refused",
Machine: "g14", Severity: Warning, Source: "test", Summary: "s",
Headline: "openrazer at 0123abc failed", Needs: "restart it.", Explanation: "It failed.",
Actions: []Action{silence}})
if err != nil {
t.Fatal(err)
}
if c.Needs != FallbackNeeds || !strings.HasPrefix(c.Explanation, NeedsYou+" "+FallbackNeeds) ||
len(c.Actions) != 1 || c.Actions[0].Label != silence.Label || strings.Contains(c.Headline, "0123abc") {
t.Errorf("the verdict or the answers were lost: %+v", c)
}
// Words that needed nothing stay "Nothing for you to do.".
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "y", Kind: "test-refused",
Machine: "g14", Severity: Warning, Source: "test", Summary: "s", Headline: "at 0123abc", Explanation: "x."})
if c.Needs != "" || !strings.HasPrefix(c.Explanation, NothingToDo) {
t.Errorf("%+v", c)
}
}
// **Words that now need the operator are said as a change** (ADR 0258): a condition whose needs or answers
// change emits condition-changed, so a warning kept quiet is said.
func TestAChangeOfWordsIsSaid(t *testing.T) {
k, _, told, _ := keeper(t)
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
if _, err := k.Observe(t.Context(), o); err != nil {
t.Fatal(err)
}
if _, err := k.Observe(t.Context(), o); err != nil { // the same words: no event
t.Fatal(err)
}
o.Needs = "check that ace is on."
if _, err := k.Observe(t.Context(), o); err != nil {
t.Fatal(err)
}
said := settled(t, told, 2)
if len(said) != 2 || said[1].Change != ChangeWords || said[1].Event != EventChanged || said[1].Needs != o.Needs {
t.Fatalf("%+v", said)
}
}
+4
View File
@@ -240,8 +240,12 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
// The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
wasNeeds, wasActions := c.Needs, c.Actions
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
escalatedWords(&c)
if c.Needs != wasNeeds || !sameActions(c.Actions, wasActions) {
changes = append(changes, Event{Change: ChangeWords, Was: wasNeeds})
}
if o.Machine != "" {
c.Subject.Machine = o.Machine
}
+3
View File
@@ -28,6 +28,9 @@ type ResourceHealth struct {
// (ADR 0240 Phase B, to-be 48 §6).
Check string `json:"check,omitempty"`
Needs string `json:"needs,omitempty"`
// Account is the account whose own service manager runs it, or the account a resource of kind account
// is (novox/hq ADR 0254).
Account string `json:"account,omitempty"`
}
// NodeHealth is a machine's newest statement, as kept.
+9
View File
@@ -143,6 +143,15 @@ type PlanGate struct {
Carried []CarriedMove `json:"carried,omitempty"`
// Failing names the modules the last judging found wanting.
Failing []string `json:"failing,omitempty"`
// Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged,
// or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's.
Healthy map[string]int `json:"healthy,omitempty"`
// Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never
// a reason to fail.
Waits map[string]string `json:"waits,omitempty"`
// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
// pass, and is not put back.
Passing []string `json:"passing,omitempty"`
}
// CarriedMove is one module's build moving on a machine with a gated send.
+13
View File
@@ -441,6 +441,15 @@ type Health struct {
// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
const KindUnit = "unit"
// KindAccount is an account a module puts in a group (novox/hq ADR 0252): healthy when the account's running
// session has every group the module declares, or nobody is logged in; unhealthy otherwise, its reason
// starting ReasonRelogin when only a new login is missing.
const KindAccount = "account"
// ReasonRelogin starts the reason of an account whose running session began before it was put in a group
// (ADR 0252): the build did what it should, and a person has one step left (novox/hq ADR 0254).
const ReasonRelogin = "relogin needed"
// The states of a machine's service managers (issue 315).
const (
UnitsRunning = "running"
@@ -529,6 +538,10 @@ type ResourceHealth struct {
// alone; Needs is the provision the check exercises (to-be 48 §6).
Check string `json:"check,omitempty"`
Needs string `json:"needs,omitempty"`
// Account is the account whose own service manager runs it, for a unit or service in an account's
// manager, and the account itself for a resource of kind KindAccount (novox/hq ADR 0254). Empty from an
// engine older than that, and for anything the machine's own manager or runtime runs.
Account string `json:"account,omitempty"`
}
// HealthSaid is the health event's body: the machine and its statement. The machine is read from the