Compare commits
4
Commits
428bacc1b0
...
bd25807ca7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bd25807ca7 | ||
|
|
c71af17421 | ||
|
|
45ae1d0112 | ||
|
|
76cfbac7a1 |
+75
-36
@@ -442,11 +442,12 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
words[node] = aboutTheMachine(node, moved, *g.Since, facts)
|
||||
}
|
||||
// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
|
||||
// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
|
||||
// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
|
||||
// machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send
|
||||
// leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the
|
||||
// settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until
|
||||
// the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is
|
||||
// put back with what failed, so none is left on the machine unjudged for other walks to wait on.
|
||||
worst, why := healthGood, ""
|
||||
var failing []string
|
||||
broken := map[string]bool{}
|
||||
reading := map[string]health{}
|
||||
waits := map[string]string{}
|
||||
var modules []string
|
||||
@@ -465,18 +466,21 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if _, seen := reading[j.module]; !seen {
|
||||
modules = append(modules, j.module)
|
||||
}
|
||||
if h == healthBroken && !slices.Contains(g.Broken, j.module) {
|
||||
g.Broken = append(g.Broken, j.module)
|
||||
if g.BrokenWhy == "" {
|
||||
g.BrokenWhy = said
|
||||
}
|
||||
}
|
||||
if slices.Contains(g.Broken, j.module) {
|
||||
h = healthBroken // found broken once, broken for the rest of the judging
|
||||
}
|
||||
if h > reading[j.module] {
|
||||
reading[j.module] = h
|
||||
}
|
||||
if h == healthPerson {
|
||||
waits[j.module] = joinSaid(waits[j.module], said)
|
||||
}
|
||||
if h > healthPerson && !slices.Contains(failing, j.module) {
|
||||
failing = append(failing, j.module)
|
||||
}
|
||||
if h == healthBroken {
|
||||
broken[j.module] = true
|
||||
}
|
||||
if h > worst {
|
||||
worst, why = h, said
|
||||
} else if h == worst && h > healthPerson && why == "" {
|
||||
@@ -486,54 +490,77 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
if g.HealthyAt == nil {
|
||||
g.HealthyAt = map[string]time.Time{}
|
||||
}
|
||||
g.Waits = nil
|
||||
var failing []string
|
||||
for _, m := range modules {
|
||||
switch {
|
||||
case reading[m] > healthPerson:
|
||||
if reading[m] > healthPerson {
|
||||
failing = append(failing, m)
|
||||
g.Healthy[m] = 0
|
||||
default:
|
||||
delete(g.HealthyAt, m)
|
||||
continue
|
||||
}
|
||||
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
|
||||
// often while another module beside it is not yet healthy counts no faster.
|
||||
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
|
||||
g.Healthy[m]++
|
||||
if w := waits[m]; w != "" {
|
||||
if g.Waits == nil {
|
||||
g.Waits = map[string]string{}
|
||||
}
|
||||
g.Waits[m] = w
|
||||
g.HealthyAt[m] = now
|
||||
}
|
||||
if w := waits[m]; w != "" {
|
||||
if g.Waits == nil {
|
||||
g.Waits = map[string]string{}
|
||||
}
|
||||
g.Waits[m] = w
|
||||
}
|
||||
}
|
||||
// passedAlone is every module that passed on its own while the send as a whole did not.
|
||||
passedAlone := func() []string {
|
||||
var out []string
|
||||
if now.Sub(*g.Since) < gateSettle {
|
||||
return nil
|
||||
}
|
||||
settled := now.Sub(*g.Since) >= gateSettle
|
||||
passedAlone := func(m string) bool {
|
||||
return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
|
||||
}
|
||||
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
|
||||
fail := func(why string) {
|
||||
g.Passing, g.Failing = nil, nil
|
||||
for _, m := range modules {
|
||||
if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
|
||||
out = append(out, m)
|
||||
if passedAlone(m) {
|
||||
g.Passing = append(g.Passing, m)
|
||||
} else {
|
||||
g.Failing = append(g.Failing, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
}
|
||||
pastBound := now.Sub(*g.Since) > gateBound
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
|
||||
g.Failing, g.Passing = failing, passedAlone()
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
var judging []string
|
||||
for _, m := range modules {
|
||||
if reading[m] != healthBroken && !passedAlone(m) {
|
||||
judging = append(judging, m)
|
||||
}
|
||||
}
|
||||
if len(judging) == 0 || pastBound {
|
||||
fail(g.BrokenWhy)
|
||||
break
|
||||
}
|
||||
// What broke fails the send; what is beside it is judged to its own verdict first, within the bound.
|
||||
g.Passes, g.LastPass, g.Failing = 0, nil, failing
|
||||
g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", "))
|
||||
case worst == healthWaiting:
|
||||
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
|
||||
// the provider's own condition says what is wrong (ADR 0240 rule 5).
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
|
||||
if now.Sub(*g.Since) > gateBound {
|
||||
g.Passing = passedAlone()
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
|
||||
if pastBound {
|
||||
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
|
||||
}
|
||||
default:
|
||||
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
|
||||
g.Passes++
|
||||
g.LastPass, g.Last, g.Failing = &now, "", nil
|
||||
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
|
||||
if g.Passes >= gatePasses && settled {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
|
||||
}
|
||||
@@ -541,6 +568,18 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
|
||||
// for a person, never another's (issue 318 review).
|
||||
func whyFor(g *inventory.PlanGate, module string) string {
|
||||
why, _, _ := strings.Cut(g.Why, waitsPrefix)
|
||||
if w := g.Waits[module]; w != "" {
|
||||
why += waitsPrefix + w
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
const waitsPrefix = "; and it waits for a person: "
|
||||
|
||||
// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
|
||||
func waitsSaid(waits map[string]string) string {
|
||||
if len(waits) == 0 {
|
||||
@@ -555,7 +594,7 @@ func waitsSaid(waits map[string]string) string {
|
||||
for _, m := range modules {
|
||||
said = append(said, waits[m])
|
||||
}
|
||||
return "; and it waits for a person: " + strings.Join(said, "; ")
|
||||
return waitsPrefix + strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
func joinSaid(a, b string) string {
|
||||
@@ -579,7 +618,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
g := state.Gate
|
||||
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
|
||||
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
|
||||
Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since})
|
||||
if err != nil && state.Build != "" {
|
||||
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
|
||||
// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
|
||||
// wait, and a provider waiting for a login says who waits on it.
|
||||
|
||||
// withGateBounds sets the gate's bounds for one test.
|
||||
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
|
||||
t.Helper()
|
||||
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
||||
gateSettle, gateEvery, gateBound = settle, every, bound
|
||||
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
||||
}
|
||||
|
||||
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
|
||||
// of its modules' resources.
|
||||
func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
|
||||
return func() gateFacts {
|
||||
now := time.Now()
|
||||
at := now
|
||||
return gateFacts{now: now,
|
||||
reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
|
||||
engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{},
|
||||
served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
|
||||
health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func healthyContainer(module string) inventory.ResourceHealth {
|
||||
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateHealthy}
|
||||
}
|
||||
|
||||
// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
|
||||
since := time.Now().Add(-time.Second)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"))
|
||||
wasGather := gatherGateFacts
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
|
||||
f := base()
|
||||
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
|
||||
At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
|
||||
return f, nil
|
||||
}
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
|
||||
pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
|
||||
judgings := 0
|
||||
for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
|
||||
if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
judgings++
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
if g.Verdict != inventory.GateFailed || judgings < gatePasses {
|
||||
t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
|
||||
judgings, g)
|
||||
}
|
||||
if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
|
||||
t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
|
||||
}
|
||||
if !strings.Contains(g.Why, "witness") {
|
||||
t.Errorf("the verdict does not say what broke: %q", g.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// broker_runtime is the node tools' module name, a core component a witness judges.
|
||||
const broker_runtime = "node-tools"
|
||||
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, 0, time.Hour, time.Hour)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
|
||||
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
|
||||
wasGather := gatherGateFacts
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
|
||||
pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
|
||||
now := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
|
||||
t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
|
||||
}
|
||||
}
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
since := time.Now().Add(-time.Minute)
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
|
||||
Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
|
||||
Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
|
||||
Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
|
||||
{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
|
||||
passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
|
||||
app, _, _ := inv.GateOf(ctx, "build-app-c2")
|
||||
late, _, _ := inv.GateOf(ctx, "build-late-c2")
|
||||
if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
|
||||
t.Errorf("app's pass: %+v; want it without late's wait", app)
|
||||
}
|
||||
if !strings.Contains(late.Why, "relogin needed on laptop") {
|
||||
t.Errorf("late's pass: %+v; want its own wait", late)
|
||||
}
|
||||
}
|
||||
|
||||
// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
|
||||
for _, b := range []inventory.Build{
|
||||
{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
|
||||
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
||||
{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
|
||||
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
||||
} {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
|
||||
b.Manifest = manifest
|
||||
b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
|
||||
Asked: b.Asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.Commit == "l1" {
|
||||
if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
|
||||
"late": "l1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
wasMoves := machineMoves
|
||||
t.Cleanup(func() { machineMoves = wasMoves })
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
gather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := gather(ctx, open, component)
|
||||
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
|
||||
State: link.StateUnhealthy, Reason: "down"}}}}
|
||||
return f, err
|
||||
}
|
||||
gateEvery, gateBound = 0, 400*time.Millisecond
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
|
||||
advancePlans(ctx, g.open)
|
||||
if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
|
||||
break
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
|
||||
!strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
|
||||
t.Fatal("late's build is not marked failed at its gate")
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
|
||||
t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
|
||||
current["late"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
|
||||
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := h0
|
||||
say := func(machine string, rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
|
||||
unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
|
||||
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
for look := 0; look < 2; look++ {
|
||||
say("anchor", account, unit)
|
||||
say("laptop", shop)
|
||||
}
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
var c conditions.Condition
|
||||
for _, x := range list {
|
||||
keys = append(keys, x.Key)
|
||||
if x.Key == "module.db.anchor.relogin-needed" {
|
||||
c = x
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
|
||||
t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
|
||||
}
|
||||
if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
|
||||
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -65,8 +66,10 @@ var handActVerbs = []handActVerb{
|
||||
// not trusted with it — either is a repair the owner should have made.
|
||||
{Verb: "plans go"},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending.
|
||||
{Verb: "conditions silence"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
{Verb: "conditions silence", Decision: "the operator's answer on a notification is their decision, " +
|
||||
"not a repair (ADR 0258)", DecidedFor: []string{conditions.CauseOperatorAnswer}},
|
||||
// An act done outside the mesh: the mesh cannot tell a repair from a decision there, so it counts —
|
||||
// except a drill recorded through it before `hand-act drill` existed (2026-10-07). It refuses the
|
||||
// cause since, so no act recorded through it now carries it.
|
||||
|
||||
@@ -152,14 +152,23 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
}
|
||||
sort.Strings(modules)
|
||||
seen := map[string]bool{}
|
||||
// became is, per module, the kind of condition this statement says of it: what a standing one of the
|
||||
// other kind turned into.
|
||||
became := map[string]string{}
|
||||
heldOn := map[string]string{}
|
||||
providers := map[catalogue.Chosen]bool{}
|
||||
for _, m := range modules {
|
||||
// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
|
||||
// operator, never urgent, cleared on the first statement that no longer says it.
|
||||
if said, waits := personWait(m, node, unhealthy[m]); waits {
|
||||
o := reloginObservation(m, node, said, unhealthy[m])
|
||||
o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
|
||||
// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
|
||||
// consumers are held under it, and its own condition is where they are said.
|
||||
if hold != nil {
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindReloginNeeded
|
||||
if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue
|
||||
}
|
||||
@@ -176,14 +185,10 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
providers[p] = true
|
||||
continue
|
||||
}
|
||||
if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
became[m] = kindModuleUnhealthy
|
||||
c, isOpen := standing[o.Key()]
|
||||
if streaks[m] < moduleUnhealthyAfter && !isOpen {
|
||||
continue // unconfirmed: one statement can be wrong; `node show` lists it
|
||||
@@ -207,7 +212,18 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
if on, held := heldOn[key]; held {
|
||||
why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
|
||||
}
|
||||
if _, err := k.Clear(ctx, key, why); err != nil {
|
||||
// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
|
||||
// line says what it became.
|
||||
resolved := ""
|
||||
switch {
|
||||
case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
|
||||
why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
|
||||
case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
|
||||
why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
|
||||
resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
|
||||
}
|
||||
if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
@@ -229,7 +245,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
|
||||
func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
|
||||
var raisedAt *conditions.Condition
|
||||
for i, c := range hold.open {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
|
||||
if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
|
||||
raisedAt = &hold.open[i]
|
||||
}
|
||||
}
|
||||
@@ -246,42 +262,104 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
|
||||
return nil
|
||||
}
|
||||
o := moduleUnhealthyObservation(p.Module, p.Node, rs)
|
||||
if waiters := hold.waitersOn(p); len(waiters) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
if said, waits := personWait(p.Module, p.Node, rs); waits {
|
||||
o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
|
||||
}
|
||||
if o.Key() != raisedAt.Key {
|
||||
return nil // its own statement says it next
|
||||
}
|
||||
sayWaitingOn(&o, hold.waitersOn(p))
|
||||
_, err := k.Observe(ctx, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// reloginKey is a module's relogin-needed condition on a machine.
|
||||
func reloginKey(module, node string) string {
|
||||
return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
|
||||
}
|
||||
|
||||
// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
|
||||
func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
|
||||
if inv == nil {
|
||||
return ""
|
||||
}
|
||||
n, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return n.Account
|
||||
}
|
||||
|
||||
// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
|
||||
// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
|
||||
// plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
|
||||
// person in again, and a restart of the module's service would start it in the same session.
|
||||
func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
// person in again, and a restart of the module's service would start it in the same session. operator is
|
||||
// the machine's operator account, when known: the words say "your account" only for it.
|
||||
func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
o := moduleUnhealthyObservation(module, node, rs)
|
||||
o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
|
||||
w := reloginWords(module, node)
|
||||
accounts := waitingAccounts(module, rs)
|
||||
yours := operator != "" && len(accounts) > 0
|
||||
for _, a := range accounts {
|
||||
yours = yours && a == operator
|
||||
}
|
||||
w := reloginWords(module, node, yours)
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
|
||||
return o
|
||||
}
|
||||
|
||||
// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
|
||||
// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
|
||||
func reloginWords(module, node string) words {
|
||||
// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
|
||||
// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
|
||||
// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
|
||||
func reloginWords(module, node string, yours bool) words {
|
||||
if yours {
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
||||
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
||||
"cannot run until you log in again. Its update is in place and nothing was undone.",
|
||||
conditions.Capital(module), module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
||||
}
|
||||
return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
|
||||
Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
|
||||
Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
|
||||
"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
|
||||
Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
|
||||
Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
|
||||
"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
|
||||
conditions.Capital(module), node, module),
|
||||
Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
|
||||
}
|
||||
|
||||
// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
|
||||
func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
|
||||
seen[accountOf(r)] = true
|
||||
out = append(out, accountOf(r))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
|
||||
// waits on it, whether it is not working or waits for a new login.
|
||||
func sayWaitingOn(o *conditions.Observation, waiters []string) {
|
||||
if len(waiters) == 0 {
|
||||
return
|
||||
}
|
||||
o.Severity = conditions.Urgent
|
||||
o.Said += "; " + waitingWords(waiters)
|
||||
o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
|
||||
o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
|
||||
}
|
||||
|
||||
// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
|
||||
// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
|
||||
func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
|
||||
var words, said, plain []string
|
||||
needs, actions := moduleNeeds(node, rs)
|
||||
needs := moduleNeeds(node, rs)
|
||||
for _, r := range rs {
|
||||
plain = append(plain, resourcePlainWords(r))
|
||||
if r.Kind == link.KindUnit {
|
||||
@@ -305,12 +383,15 @@ func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHeal
|
||||
Explanation: fmt.Sprintf("%s on %s is not healthy: %s. It clears as soon as it runs again.", module, node,
|
||||
namesWords(plain, 3)),
|
||||
Needs: needs,
|
||||
Actions: actions,
|
||||
Resolved: fmt.Sprintf("%s works again on %s", module, node)}
|
||||
}
|
||||
|
||||
// reasonWords is why a resource is unhealthy, as a person reads it.
|
||||
func reasonWords(r inventory.ResourceHealth) string {
|
||||
// An account waiting for a new login (ADR 0252) is said in the mesh's words, not the engine's.
|
||||
if r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) {
|
||||
return fmt.Sprintf("waits for a new login of %s, which is in the group and logged in before it was", accountOf(r))
|
||||
}
|
||||
switch r.Reason {
|
||||
case "restarting":
|
||||
return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
|
||||
@@ -353,18 +434,14 @@ const kindReloginNeeded = "relogin-needed"
|
||||
// not in its group or that could not be read — is not a wait, and the module is judged as before. A
|
||||
// resource still starting is not unhealthy and does not make a wait either. Pure.
|
||||
func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
|
||||
waiting := map[string]bool{}
|
||||
var accounts []string
|
||||
for _, r := range rs {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
|
||||
waiting[accountOf(r)] = true
|
||||
accounts = append(accounts, accountOf(r))
|
||||
}
|
||||
}
|
||||
accounts := waitingAccounts(module, rs)
|
||||
if len(accounts) == 0 {
|
||||
return "", false
|
||||
}
|
||||
waiting := map[string]bool{}
|
||||
for _, a := range accounts {
|
||||
waiting[a] = true
|
||||
}
|
||||
var units []string
|
||||
for _, r := range rs {
|
||||
if r.Module != module || r.State != link.StateUnhealthy {
|
||||
@@ -378,13 +455,12 @@ func personWait(module, machine string, rs []inventory.ResourceHealth) (string,
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
sort.Strings(accounts)
|
||||
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
|
||||
"running session began before it was; log out of every session and in again, or reboot", machine, module,
|
||||
"login began before it was; log out of %[1]s completely and log in again, or restart it", machine, module,
|
||||
strings.Join(accounts, ", "))
|
||||
if len(units) > 0 {
|
||||
sort.Strings(units)
|
||||
said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
|
||||
said += fmt.Sprintf(" (until then %s cannot run)", strings.Join(units, ", "))
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
@@ -416,6 +492,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
|
||||
// account already waiting before this send was not brought to it by this build, and a later build of
|
||||
// the module sent while the login is still owed is judged as before. Two seconds of skew between the
|
||||
// machine's clock and the controller's, as the witness allows.
|
||||
if waits {
|
||||
for _, r := range h.Resources {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
|
||||
waits = false
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
continue
|
||||
|
||||
@@ -75,11 +75,24 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
account := relogin("lights", "operator")
|
||||
account.Since = since.Add(10 * time.Second) // the send put it in its wait
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
|
||||
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}}
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
|
||||
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
// **Only the send that put the account in its wait is excused** (issue 318 review): an account waiting
|
||||
// since before the send is judged as before, so a later build of the module is not passed on an old wait.
|
||||
older := f.health["laptop"]
|
||||
older.Resources = []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}
|
||||
older.Resources[0].Since = since.Add(-time.Hour)
|
||||
f.health["laptop"] = older
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
|
||||
t.Fatalf("a wait from before the send reads %v %q; want not yet", h, why)
|
||||
}
|
||||
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}
|
||||
h := f.health["laptop"]
|
||||
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
|
||||
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
|
||||
@@ -197,7 +210,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||
// no button, since nothing but their own new login can do it.
|
||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
|
||||
@@ -206,9 +219,64 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
if len(o.Actions) != 0 || o.Needs == "" {
|
||||
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
|
||||
}
|
||||
// The kind's own wording, for a condition raised without words, says the same.
|
||||
if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs ||
|
||||
w.Headline != o.Headline || len(w.Actions) != 0 {
|
||||
// **Not "your account" when it is not the operator's** (issue 318 review).
|
||||
other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "guest"), userUnit("openrazer", "guest")})
|
||||
plainExample(t, other, "openrazer waits for a new login on g14",
|
||||
"Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+
|
||||
"put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+
|
||||
"until it logs in again. Its update is in place and nothing was undone.")
|
||||
if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") {
|
||||
t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation)
|
||||
}
|
||||
// The kind's own wording, for a condition raised without words, names the module whole, dots and all,
|
||||
// and never claims the account is the operator's.
|
||||
w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"})
|
||||
if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 ||
|
||||
strings.Contains(w.Explanation, "your account") {
|
||||
t.Errorf("the kind's wording: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
||||
// became, never that it works again (issue 318 review).
|
||||
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
store := conditions.NewInMemory()
|
||||
told := &conditions.Told{}
|
||||
k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told})
|
||||
t.Cleanup(func() { k.Close(context.Background()) })
|
||||
at := h0
|
||||
say := func(rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
|
||||
Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
|
||||
account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator",
|
||||
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"}
|
||||
say(unit)
|
||||
say(unit) // lights not working
|
||||
say(account, unit)
|
||||
say(account, unit) // now it waits for a login
|
||||
var resolved []string
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
for _, e := range told.Said() {
|
||||
if e.Change == conditions.ChangeCleared {
|
||||
resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") {
|
||||
t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,11 +184,12 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
Resolved: conditions.Capital(thing) + " works again"}
|
||||
}),
|
||||
kindReloginNeeded: worded(func(o conditions.Observation) words {
|
||||
// A module's name may hold dots: it is the id without its machine.
|
||||
module := ""
|
||||
if o.Scope == conditions.ScopeModule {
|
||||
module = idPart(o, 0)
|
||||
if o.Scope == conditions.ScopeModule && o.Machine != "" {
|
||||
module = strings.TrimSuffix(o.ID, "."+o.Machine)
|
||||
}
|
||||
return reloginWords(orModule(module), machineOr(o, "a machine"))
|
||||
return reloginWords(orModule(module), machineOr(o, "a machine"), false)
|
||||
}),
|
||||
kindProviderFailing: worded(func(o conditions.Observation) words {
|
||||
thing, consumer := conditions.ThingWords(o), idPart(o, 2)
|
||||
@@ -642,43 +643,28 @@ func waitingNeeds(severity conditions.Severity) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// waitingActions are the answers to a walk waiting past its urgent bound: the controller's own verb, since
|
||||
// the module that should have said go is the one not answering.
|
||||
func waitingActions(w waitFacts, severity conditions.Severity) []conditions.Action {
|
||||
if severity != conditions.Urgent {
|
||||
return nil
|
||||
}
|
||||
return []conditions.Action{
|
||||
{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": w.id, "why": ""}},
|
||||
{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": w.id, "why": ""}},
|
||||
}
|
||||
}
|
||||
|
||||
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
|
||||
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
|
||||
func moduleNeeds(node string, rs []inventory.ResourceHealth) (string, []conditions.Action) {
|
||||
var actions []conditions.Action
|
||||
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
|
||||
func moduleNeeds(node string, rs []inventory.ResourceHealth) string {
|
||||
unit := ""
|
||||
for _, r := range rs {
|
||||
if strings.Contains(r.Reason, "relogin needed") {
|
||||
return fmt.Sprintf("log out of every session on %s and log in again.", node), nil
|
||||
return reloginNeeds(node)
|
||||
}
|
||||
if r.Kind == link.KindUnit && len(actions) < 2 {
|
||||
scope := "system"
|
||||
if strings.Contains(r.Reason, "account's own") {
|
||||
scope = "user"
|
||||
}
|
||||
label := "Restart"
|
||||
if len(actions) > 0 {
|
||||
label = "Restart " + unitPlainWords(r.Target)
|
||||
}
|
||||
actions = append(actions, conditions.Action{Label: label, Verb: "node-service-manager.restart",
|
||||
Machine: node, Arguments: map[string]string{"unit": r.Target, "scope": scope}})
|
||||
if r.Kind == link.KindUnit && unit == "" {
|
||||
unit = unitPlainWords(r.Target)
|
||||
}
|
||||
}
|
||||
if len(actions) > 0 {
|
||||
return "restart it; if it fails again, the details say why.", actions
|
||||
if unit != "" {
|
||||
return fmt.Sprintf("restart its service %s on %s; if it fails again, the details say why.", unit, node)
|
||||
}
|
||||
return "", nil
|
||||
return ""
|
||||
}
|
||||
|
||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||
func reloginNeeds(node string) string {
|
||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||
}
|
||||
|
||||
// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it.
|
||||
@@ -695,18 +681,15 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
|
||||
long = "for " + humanDuration(d)
|
||||
}
|
||||
if o.Resolver == conditions.ResolverOperator {
|
||||
// Words only: releasing or stopping a delivery is not an acknowledgement, so no desk click
|
||||
// performs it (ADR 0258).
|
||||
switch held {
|
||||
case "held":
|
||||
needs = "release it, or stop it."
|
||||
actions = []conditions.Action{
|
||||
{Label: "Release", Verb: "mesh-delivery.release", Arguments: map[string]string{"id": l.ID, "why": ""}},
|
||||
{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}},
|
||||
}
|
||||
case "ready", "checked":
|
||||
needs = "merge its pull request, or close it."
|
||||
default:
|
||||
needs = "stop it, or read the details to see what it waits for."
|
||||
actions = []conditions.Action{{Label: "Stop", Verb: "mesh-delivery.stop", Arguments: map[string]string{"id": l.ID, "why": ""}}}
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("Delivery of %s %s %s", name, held, long),
|
||||
|
||||
@@ -70,13 +70,7 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
|
||||
plainExample(t, got[0], "openrazer delivery waiting to start",
|
||||
"Needs you: start it, or stop it. The change to openrazer is merged and built, and mesh-delivery (the "+
|
||||
"module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+
|
||||
"be stuck.", "Start", "Stop")
|
||||
if a := got[0].Actions[0]; a.Verb != "mesh-controller.plans" || a.Arguments["go"] != "plan-1791454185265004861" {
|
||||
t.Errorf("start: %+v", a)
|
||||
}
|
||||
if a := got[0].Actions[1]; a.Verb != "mesh-controller.plans" || a.Arguments["stop"] != "plan-1791454185265004861" {
|
||||
t.Errorf("stop: %+v", a)
|
||||
}
|
||||
"be stuck.")
|
||||
|
||||
// Many modules are counted, not listed in the headline.
|
||||
f.waits[0].modules = []string{"a", "b", "c", "d"}
|
||||
@@ -87,24 +81,35 @@ func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *test
|
||||
}
|
||||
|
||||
// **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the
|
||||
// account's own service manager (exit-code)". The operator restarts it from the notification.
|
||||
func TestAModuleUnhealthyOffersARestartOfItsService(t *testing.T) {
|
||||
// account's own service manager (exit-code)". Restarting is not an acknowledgement, so it is said in words
|
||||
// and offered as no answer (ADR 0258).
|
||||
func TestAModuleUnhealthyAsksForARestartInWords(t *testing.T) {
|
||||
o := moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: link.KindUnit,
|
||||
Resource: "openrazer-daemon", Target: "openrazer-daemon.service",
|
||||
Reason: "failed in the account's own service manager (exit-code)", Since: time.Now()}})
|
||||
plainExample(t, o, "openrazer not working on g14",
|
||||
"Needs you: restart it; if it fails again, the details say why. openrazer on g14 is not healthy: its "+
|
||||
"service openrazer-daemon stopped with an error. It clears as soon as it runs again.", "Restart")
|
||||
if a := o.Actions[0]; a.Verb != "node-service-manager.restart" || a.Machine != "g14" ||
|
||||
a.Arguments["unit"] != "openrazer-daemon.service" || a.Arguments["scope"] != "user" {
|
||||
t.Errorf("restart: %+v", a)
|
||||
}
|
||||
// An account waiting for a new login (ADR 0252) asks for the login, which no button can give.
|
||||
"Needs you: restart its service openrazer-daemon on g14; if it fails again, the details say why. "+
|
||||
"openrazer on g14 is not healthy: its service openrazer-daemon stopped with an error. It clears as soon "+
|
||||
"as it runs again.")
|
||||
// An account waiting for a new login (ADR 0252) asks for the login, held to the plain rule.
|
||||
o = moduleUnhealthyObservation("openrazer", "g14", []inventory.ResourceHealth{{Kind: "account",
|
||||
Resource: "operator-in-group", Target: "jochen", Reason: "relogin needed: the account is in the group"}})
|
||||
if o.Needs != "log out of every session on g14 and log in again." || len(o.Actions) != 0 {
|
||||
if o.Needs != "log out of g14 completely and log in again, or restart it." || len(o.Actions) != 0 {
|
||||
t.Errorf("relogin: %q %+v", o.Needs, o.Actions)
|
||||
}
|
||||
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(w, "g14"); !ok {
|
||||
t.Errorf("the relogin words are not plain: %s", why)
|
||||
}
|
||||
// And the kind issue 318 raises for it (ADR 0254).
|
||||
rw := plainWordings["relogin-needed"](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14",
|
||||
Machine: "g14", Kind: "relogin-needed", Severity: conditions.Warning})
|
||||
if why, ok := conditions.PlainWords(rw, "g14"); !ok || rw.Needs == "" || len(rw.Actions) != 0 {
|
||||
t.Errorf("relogin-needed: %s %+v", why, rw)
|
||||
}
|
||||
if strings.Contains(o.Summary, "session") || !strings.Contains(o.Summary, "waits for a new login of jochen") {
|
||||
t.Errorf("relogin summary: %q", o.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
// **Failed units on a machine**: "shanks's service manager is degraded: 3 failed unit(s) no module places —
|
||||
@@ -143,15 +148,12 @@ func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) {
|
||||
|
||||
// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been
|
||||
// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …".
|
||||
func TestADeliveryHeldOffersReleaseAndStop(t *testing.T) {
|
||||
func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) {
|
||||
got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}})
|
||||
plainExample(t, got[0], "Delivery of hq held for 36 hours",
|
||||
"Needs you: release it, or stop it. A delivery of hq has been held for 36 hours, past its limit.",
|
||||
"Release", "Stop")
|
||||
if a := got[0].Actions[0]; a.Verb != "mesh-delivery.release" || a.Arguments["id"] != "novox/hq@055550802096" {
|
||||
t.Errorf("release: %+v", a)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// **Every kind the controller raises has plain words**, and its words are plain for a subject of every
|
||||
@@ -184,3 +186,24 @@ func TestEveryWordingIsPlain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **An answer on a notification is no repair** (novox/hq ADR 0258): silencing chosen by the operator on the
|
||||
// desk does not count toward a healer wanted; the same silence by hand for another cause still does.
|
||||
func TestAnOperatorsAnswerIsNoHandRepair(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
for i := 0; i < 3; i++ {
|
||||
a := actByHand(now.Add(-time.Duration(i+1)*time.Hour), conditions.CauseOperatorAnswer)
|
||||
a.Verb = "conditions silence"
|
||||
f.handActs = append(f.handActs, a)
|
||||
}
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Fatalf("an answer counted as a repair: %+v", got)
|
||||
}
|
||||
for i := range f.handActs {
|
||||
f.handActs[i].Cause = "machine-units"
|
||||
}
|
||||
if got := watchHandActs(f); len(got) != 1 {
|
||||
t.Fatalf("a silence by hand stopped counting: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,7 +319,7 @@ func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
}
|
||||
seen[c.Build] = true
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module),
|
||||
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
|
||||
}
|
||||
@@ -375,10 +375,7 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
continue
|
||||
}
|
||||
seen[c.Build] = true
|
||||
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why)
|
||||
if w := g.Waits[c.Module]; w != "" {
|
||||
why += "; and it waits for a person: " + w
|
||||
}
|
||||
why := passedAloneWhy(g, c.Module)
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
|
||||
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
|
||||
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
|
||||
@@ -393,6 +390,15 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
return kept
|
||||
}
|
||||
|
||||
// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
|
||||
func passedAloneWhy(g *inventory.PlanGate, module string) string {
|
||||
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
|
||||
if w := g.Waits[module]; w != "" {
|
||||
why += waitsPrefix + w
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// sentTheBuild is every machine running a module that was last sent this build of it.
|
||||
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
|
||||
running, err := open.inventory.Running(ctx, module)
|
||||
|
||||
@@ -703,7 +703,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if lead.Gate.Verdict != inventory.GatePassed {
|
||||
continue
|
||||
}
|
||||
passedWith(state, state.GatedBy, lead.Gate)
|
||||
passedWith(m, state, state.GatedBy, lead.Gate)
|
||||
}
|
||||
switch {
|
||||
case step.failed != "":
|
||||
@@ -937,14 +937,14 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
|
||||
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
|
||||
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
|
||||
func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
|
||||
if s.Gate == nil {
|
||||
s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since}
|
||||
}
|
||||
if s.Gate.Verdict != "" {
|
||||
return
|
||||
}
|
||||
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took
|
||||
s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took
|
||||
s.Gate.Passes, s.Gate.Kept = g.Passes, true
|
||||
}
|
||||
|
||||
@@ -963,6 +963,18 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
|
||||
g := state.Gate
|
||||
if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
|
||||
gateFailed(batched, open, p, module, state, machines, why)
|
||||
} else if slices.Contains(g.Passing, module) && state.Build != "" {
|
||||
// **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes
|
||||
// asked while another of its send failed, its build has a verdict, and other walks do not wait on it.
|
||||
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed,
|
||||
Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil {
|
||||
fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
state.Why = "passed on its own; stopped with the send that carried it: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept",
|
||||
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
|
||||
} else {
|
||||
state.Why = "not found wanting; stopped with the send that carried it: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
|
||||
@@ -361,7 +361,6 @@ func watchWaits(f *signalFacts) []conditions.Observation {
|
||||
Headline: deliveryName(w.modules, w.repository) + " waiting to start",
|
||||
Explanation: walkWaitingWords(w, in, severity),
|
||||
Needs: waitingNeeds(severity),
|
||||
Actions: waitingActions(w, severity),
|
||||
Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"})
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -255,8 +255,13 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||
t.Fatalf("%s's build was marked failed for the node tools' condition", m)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(p.Note, "app1 was not found wanting") {
|
||||
t.Fatalf("the plan blames the module its gate was kept on: %s", p.Note)
|
||||
// Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318
|
||||
// review): it is not blamed, and not left without a verdict.
|
||||
if !strings.Contains(p.Note, "app1 passed on its own and is kept") {
|
||||
t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note)
|
||||
}
|
||||
if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("app1's own pass was not kept: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ const (
|
||||
ChangeSilenced = "silenced"
|
||||
ChangeUnsilenced = "silence-ended"
|
||||
ChangeCleared = "cleared"
|
||||
// ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR
|
||||
// 0258): a warning kept quiet as needing nothing is said once its words need the operator.
|
||||
ChangeWords = "words"
|
||||
)
|
||||
|
||||
// Event is the body of every condition event, and the shape a history entry keeps: **the condition
|
||||
|
||||
@@ -62,10 +62,37 @@ const (
|
||||
)
|
||||
|
||||
// SilenceAction is the action that stops a condition's messages for a week, with the operator's why: the
|
||||
// answer to a condition the operator decided to live with.
|
||||
// answer to a condition the operator decided to live with. It is an acknowledgement (ADR 0234 §8): the
|
||||
// only kind of answer a desk click performs until answers are authorised (novox/hq ADR 0258). Its cause
|
||||
// marks it as an answer, which the hand-act log does not count as a repair.
|
||||
func SilenceAction(key string) Action {
|
||||
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
|
||||
Arguments: map[string]string{"silence": key, "for": "7d", "why": ""}}
|
||||
Arguments: map[string]string{"silence": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
|
||||
}
|
||||
|
||||
// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with.
|
||||
const CauseOperatorAnswer = "operator-answer"
|
||||
|
||||
// FallbackNeeds is what a condition needs when its producer's words needed the operator and could not
|
||||
// be said: the verdict is kept, never turned into "Nothing for you to do." (ADR 0258).
|
||||
const FallbackNeeds = "read the details to see what to do."
|
||||
|
||||
func sameActions(a, b []Action) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i].Label != b[i].Label || a[i].Verb != b[i].Verb || a[i].Machine != b[i].Machine ||
|
||||
len(a[i].Arguments) != len(b[i].Arguments) {
|
||||
return false
|
||||
}
|
||||
for k, v := range a[i].Arguments {
|
||||
if b[i].Arguments[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Bounds of the plain words: a headline fits a notification's title line, an explanation two sentences.
|
||||
@@ -108,7 +135,9 @@ var (
|
||||
goDuration = regexp.MustCompile(`\b[0-9]+(h[0-9]+m|m[0-9]+s|h[0-9]+m[0-9]+s)\b|\b[0-9]+(\.[0-9]+)?(ns|ms|µs)\b`)
|
||||
clockTime = regexp.MustCompile(`\b[0-9]{1,2}:[0-9]{2}\b|\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b|\bUTC\b`)
|
||||
flag = regexp.MustCompile(`(^|\s)--?[a-z]`)
|
||||
agentWord = regexp.MustCompile(`(?i)\bagents?\b|\bby hand\b|\bsession\b`)
|
||||
agentWord = regexp.MustCompile(`(?i)\bagents?\b`)
|
||||
byHand = regexp.MustCompile(`(?i)\bby hand\b`)
|
||||
latinAbbr = regexp.MustCompile(`\b(e\.g|i\.e|etc)\.`)
|
||||
markup = regexp.MustCompile("[`*<>{}\\[\\]|#]|(^|\\s)_|_(\\s|$)")
|
||||
)
|
||||
|
||||
@@ -126,14 +155,15 @@ func Plain(text string, machines ...string) (string, bool) {
|
||||
if m := markup.FindString(text); m != "" {
|
||||
return "markup (" + m + ")", false
|
||||
}
|
||||
text = latinAbbr.ReplaceAllString(text, "eg")
|
||||
for _, shape := range []struct {
|
||||
re *regexp.Regexp
|
||||
what string
|
||||
}{{hexID, "a hash"}, {numberedID, "a numbered id"}, {dotted, "a dotted name"}, {goDuration, "a duration in code"},
|
||||
{clockTime, "a clock time or date"}, {flag, "a command's flag"}} {
|
||||
if m := shape.re.FindString(text); m != "" {
|
||||
if shape.re == hexID && !strings.ContainsAny(m, "0123456789") {
|
||||
continue // a word of letters a to f only
|
||||
if shape.re == hexID && (!strings.ContainsAny(m, "0123456789") || !strings.ContainsAny(m, "abcdef")) {
|
||||
continue // letters a to f only, or a number: words and counts, not a hash
|
||||
}
|
||||
return shape.what + " (" + strings.TrimSpace(m) + ")", false
|
||||
}
|
||||
@@ -158,16 +188,14 @@ func PlainWords(w Words, machines ...string) (string, bool) {
|
||||
if len(w.Explanation) > ExplanationMax {
|
||||
return fmt.Sprintf("explanation: longer than %d characters", ExplanationMax), false
|
||||
}
|
||||
for _, m := range agentWord.FindAllString(w.Explanation, -1) {
|
||||
if !strings.EqualFold(m, "by hand") { // "repaired by hand" is a fact; telling to act by hand is not
|
||||
return "explanation: sends the operator elsewhere (" + m + ")", false
|
||||
}
|
||||
if m := agentWord.FindString(w.Explanation); m != "" {
|
||||
return "explanation: sends the operator elsewhere (" + m + ")", false
|
||||
}
|
||||
if w.Needs != "" {
|
||||
if why, ok := Plain(w.Needs, machines...); !ok {
|
||||
return "needs: " + why, false
|
||||
}
|
||||
if m := agentWord.FindString(w.Needs); m != "" {
|
||||
if m := agentWord.FindString(w.Needs) + byHand.FindString(w.Needs); m != "" {
|
||||
return "needs: not something the operator does themselves (" + m + ")", false
|
||||
}
|
||||
if len(w.Needs) > NeedsMax || !strings.HasSuffix(w.Needs, ".") {
|
||||
@@ -221,6 +249,8 @@ func plainly(o Observation) Observation {
|
||||
if w.Headline != "" && w.Resolved == "" {
|
||||
w.Resolved = "Resolved: " + lowerFirst(w.Headline)
|
||||
}
|
||||
// An explanation too long is cut, never refused: what it says first is what matters.
|
||||
w.Explanation = cut(w.Explanation, ExplanationMax-len(Verdict(w.Needs, ""))-1)
|
||||
if from == "" {
|
||||
if Unworded != nil {
|
||||
Unworded(o, "the kind "+o.Kind+" has no plain words")
|
||||
@@ -230,13 +260,42 @@ func plainly(o Observation) Observation {
|
||||
if Unworded != nil {
|
||||
Unworded(o, from+" is not plain: "+why)
|
||||
}
|
||||
// The scope's words, but never a weaker verdict: what needed the operator still does, and its
|
||||
// answers are kept where they are themselves sound.
|
||||
needed, actions := w.Needs != "", soundActions(w.Actions)
|
||||
w = scopeWords(o)
|
||||
if needed {
|
||||
w.Needs, w.Actions = FallbackNeeds, actions
|
||||
}
|
||||
}
|
||||
o.Headline, o.Resolved, o.Needs, o.Actions = w.Headline, w.Resolved, w.Needs, w.Actions
|
||||
o.Explanation = Verdict(w.Needs, w.Explanation)
|
||||
return o
|
||||
}
|
||||
|
||||
// soundActions are the actions with a label and a verb.
|
||||
func soundActions(list []Action) []Action {
|
||||
var out []Action
|
||||
for _, a := range list {
|
||||
if a.Label != "" && len(a.Label) <= 24 && a.Verb != "" {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// cut shortens text to at most n bytes at a word, ending it with an ellipsis.
|
||||
func cut(text string, n int) string {
|
||||
if len(text) <= n || n < 10 {
|
||||
return text
|
||||
}
|
||||
t := text[:n-3]
|
||||
if i := strings.LastIndexByte(t, ' '); i > n/2 {
|
||||
t = t[:i]
|
||||
}
|
||||
return strings.TrimRight(t, " ,;:") + "…"
|
||||
}
|
||||
|
||||
// Verdict is an explanation opened by its verdict.
|
||||
func Verdict(needs, explanation string) string {
|
||||
if needs == "" {
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
func TestThePlainRuleRefusesIdentifiersCodeAndTimes(t *testing.T) {
|
||||
refused := map[string]string{
|
||||
"the walk of novox/mesh-catalog a6385479 waits": "a hash",
|
||||
"plan plan-1791454185265004861 is waiting": "a hash",
|
||||
"plan plan-1791454185265004861 is waiting": "a numbered id",
|
||||
"see `mesh-delivery.show` for why": "markup",
|
||||
"call mesh-delivery.show for why": "a dotted name",
|
||||
"key plan.x.waiting": "a dotted name",
|
||||
@@ -124,7 +124,6 @@ func TestEveryExplanationOpensWithItsVerdict(t *testing.T) {
|
||||
for _, w := range []Words{
|
||||
{Headline: "x waits", Explanation: "Have an agent start it.", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Needs: "start it by hand.", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Needs: "open a session and start it.", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Needs: "start it", Resolved: "x started"},
|
||||
{Headline: "x waits", Explanation: "It waits.", Resolved: "x started", Actions: []Action{{Label: "Start"}}},
|
||||
} {
|
||||
@@ -161,3 +160,88 @@ func TestAnEscalatedConditionSaysItNeedsTheOperator(t *testing.T) {
|
||||
t.Errorf("seen again after escalation: %q", c.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// **What the plain rule refused wrongly** (novox/hq ADR 0258): a login session is the operator's own word,
|
||||
// "e.g." is not a dotted name, a long count is not a hash, and a long explanation is cut, not refused.
|
||||
func TestThePlainRuleRefusesNoPlainWords(t *testing.T) {
|
||||
for _, text := range []string{
|
||||
"log out of every session on g14 and log in again.",
|
||||
"A module, e.g. the lighting daemon, waits.",
|
||||
"It holds 12345678 files.",
|
||||
"A person repaired push by hand 35 times in 14 days.",
|
||||
} {
|
||||
if why, ok := Plain(text, "g14"); !ok {
|
||||
t.Errorf("%q refused: %s", text, why)
|
||||
}
|
||||
}
|
||||
if _, ok := Plain("commit 0123abc landed"); ok {
|
||||
t.Error("a hash passed")
|
||||
}
|
||||
k, _, _, _ := keeper(t)
|
||||
long := strings.Repeat("The bus refused a message. ", 30)
|
||||
Wording("test-long", func(o Observation) Words {
|
||||
return Words{Headline: "A long one", Explanation: long, Resolved: "Done"}
|
||||
})
|
||||
before := Unworded
|
||||
var unworded []string
|
||||
Unworded = func(o Observation, why string) { unworded = append(unworded, why) }
|
||||
t.Cleanup(func() { Unworded = before })
|
||||
c, err := k.Observe(t.Context(), Observation{Scope: ScopeBus, ID: "x", Kind: "test-long", Severity: Warning,
|
||||
Source: "test", Summary: "long"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Headline != "A long one" || len(c.Explanation) > ExplanationMax || !strings.HasSuffix(c.Explanation, "…") ||
|
||||
len(unworded) != 0 {
|
||||
t.Errorf("a long explanation was not cut: %d %q %v", len(c.Explanation), c.Headline, unworded)
|
||||
}
|
||||
}
|
||||
|
||||
// **Words refused keep their verdict and their answers** (ADR 0258): a producer that needed the operator
|
||||
// still does, said as "read the details", and its sound answers stay.
|
||||
func TestRefusedWordsKeepTheirVerdictAndAnswers(t *testing.T) {
|
||||
k, _, _, _ := keeper(t)
|
||||
before := Unworded
|
||||
Unworded = func(Observation, string) {}
|
||||
t.Cleanup(func() { Unworded = before })
|
||||
silence := SilenceAction("module.openrazer.g14.x")
|
||||
c, err := k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "x", Kind: "test-refused",
|
||||
Machine: "g14", Severity: Warning, Source: "test", Summary: "s",
|
||||
Headline: "openrazer at 0123abc failed", Needs: "restart it.", Explanation: "It failed.",
|
||||
Actions: []Action{silence}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Needs != FallbackNeeds || !strings.HasPrefix(c.Explanation, NeedsYou+" "+FallbackNeeds) ||
|
||||
len(c.Actions) != 1 || c.Actions[0].Label != silence.Label || strings.Contains(c.Headline, "0123abc") {
|
||||
t.Errorf("the verdict or the answers were lost: %+v", c)
|
||||
}
|
||||
// Words that needed nothing stay "Nothing for you to do.".
|
||||
c, _ = k.Observe(t.Context(), Observation{Scope: ScopeModule, ID: "openrazer.g14", Token: "y", Kind: "test-refused",
|
||||
Machine: "g14", Severity: Warning, Source: "test", Summary: "s", Headline: "at 0123abc", Explanation: "x."})
|
||||
if c.Needs != "" || !strings.HasPrefix(c.Explanation, NothingToDo) {
|
||||
t.Errorf("%+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// **Words that now need the operator are said as a change** (ADR 0258): a condition whose needs or answers
|
||||
// change emits condition-changed, so a warning kept quiet is said.
|
||||
func TestAChangeOfWordsIsSaid(t *testing.T) {
|
||||
k, _, told, _ := keeper(t)
|
||||
o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning, Source: "S1",
|
||||
Summary: "ace is silent", Headline: "ace is not answering", Explanation: "It may be asleep."}
|
||||
if _, err := k.Observe(t.Context(), o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.Observe(t.Context(), o); err != nil { // the same words: no event
|
||||
t.Fatal(err)
|
||||
}
|
||||
o.Needs = "check that ace is on."
|
||||
if _, err := k.Observe(t.Context(), o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settled(t, told, 2)
|
||||
if len(said) != 2 || said[1].Change != ChangeWords || said[1].Event != EventChanged || said[1].Needs != o.Needs {
|
||||
t.Fatalf("%+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,8 +240,12 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
// The kind as the source says it now: a source that gave the same key a kind of its own since
|
||||
// (a probe's finding split out for a healer) is read by that kind from its next observation.
|
||||
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
|
||||
wasNeeds, wasActions := c.Needs, c.Actions
|
||||
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions
|
||||
escalatedWords(&c)
|
||||
if c.Needs != wasNeeds || !sameActions(c.Actions, wasActions) {
|
||||
changes = append(changes, Event{Change: ChangeWords, Was: wasNeeds})
|
||||
}
|
||||
if o.Machine != "" {
|
||||
c.Subject.Machine = o.Machine
|
||||
}
|
||||
@@ -276,6 +280,13 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
|
||||
|
||||
// Clear removes a condition an observation says is resolved, and says so. False when none was open.
|
||||
func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||
return k.ClearSaying(ctx, key, why, "")
|
||||
}
|
||||
|
||||
// ClearSaying is Clear with the line the operator reads when it clears, in place of the condition's own
|
||||
// resolved line: for a condition that ends because it became another (novox/hq ADR 0254: a module not
|
||||
// working that now waits for a new login is not "working again"). Empty keeps the condition's own.
|
||||
func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bool, error) {
|
||||
for i := 0; i < tries; i++ {
|
||||
entry, found, err := k.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
@@ -292,6 +303,9 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
|
||||
if err := k.stamp(&c); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if resolved != "" {
|
||||
c.Resolved = resolved
|
||||
}
|
||||
if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
|
||||
@@ -152,6 +152,13 @@ type PlanGate struct {
|
||||
// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
|
||||
// pass, and is not put back.
|
||||
Passing []string `json:"passing,omitempty"`
|
||||
// HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery
|
||||
// after the one before (issue 318 review).
|
||||
HealthyAt map[string]time.Time `json:"healthy_at,omitempty"`
|
||||
// Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the
|
||||
// first reason: the send fails, and the modules beside them are judged to their own verdict first.
|
||||
Broken []string `json:"broken,omitempty"`
|
||||
BrokenWhy string `json:"broken_why,omitempty"`
|
||||
}
|
||||
|
||||
// CarriedMove is one module's build moving on a machine with a gated send.
|
||||
|
||||
Reference in New Issue
Block a user