Put a broken module back at once, and excuse a wait only for a move that added an account group (hq issue 318 review)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk

This commit is contained in:
jochen
2026-10-08 15:25:58 +02:00
parent 7ad9dbcb5d
commit a63939160e
8 changed files with 348 additions and 26 deletions
+37 -1
View File
@@ -119,6 +119,9 @@ type gateFacts struct {
healthErr error
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
heldOn map[string]string
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
// not (issue 318 review): the only move whose wait for a new login is excused.
groupsAdded map[string]bool
}
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
@@ -430,6 +433,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
if err != nil {
return "", err
}
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
// gate happens to be kept on.
@@ -544,7 +548,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
fail(g.BrokenWhy)
break
}
// What broke fails the send; what is beside it is judged to its own verdict first, within the bound.
// What broke fails the send, and is put back at once by the caller (putBackBroken); what is beside
// it is judged to its own verdict first, within the bound.
g.Passes, g.LastPass, g.Failing = 0, nil, failing
g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", "))
case worst == healthWaiting:
@@ -607,6 +612,37 @@ func joinSaid(a, b string) string {
return a + "; " + b
}
// movesAddingGroups is, per module a gate judges, whether its move puts an account in a group the build it
// moved from did not: read from the builds' manifests. A module whose move is not known adds none.
func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
shelf map[string]catalogue.Manifest) map[string]bool {
out := map[string]bool{}
for _, j := range pairs {
if _, done := out[j.module]; done {
continue
}
from, to := g.From, g.To
for _, c := range g.Carried {
if c.Module == j.module {
from, to = c.From, c.To
break
}
}
target, found, err := inv.ManifestAt(ctx, j.module, to)
if err != nil || !found {
target = shelf[j.module]
}
before, had, err := inv.ManifestAt(ctx, j.module, from)
if err != nil || (from != "" && !had) {
// What it moved from is not known: no wait is excused, rather than one the move did not bring.
out[j.module] = false
continue
}
out[j.module] = addsAccountGroups(before, had, target)
}
return out
}
// decide sets a gate's verdict.
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
+144
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
"time"
@@ -277,3 +278,146 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
}
}
// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
// tools are marked and put back in the very judging that found them broken — their registered build is the
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
build := func(module, commit string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"app", broker_runtime} {
build(m, "c1", old)
if _, err := inv.Assign(ctx, "laptop", m); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
t.Fatal(err)
}
build("app", "c2", recent)
build(broker_runtime, "c2", recent)
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
})
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, _ := open.inventory.Assigned(ctx, node)
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
var sends []string
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
n++
carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
sends = append(sends, node+":"+carried[broker_runtime])
digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
var seen []string // the node tools' registered build at each judging
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
now := time.Now()
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, _ := open.inventory.LastReports(ctx)
for _, r := range reports {
f.reports[r.Node] = r
}
current, _ := open.inventory.CurrentBuilds(ctx)
seen = append(seen, current[broker_runtime].Commit)
app := healthyContainer("app")
if current[broker_runtime].Commit == "c2" {
// The witness reverted the node tools; app cannot reach them yet.
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
app.State, app.Reason = link.StateUnhealthy, "down"
}
f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
return f, nil
}
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
release := func() inventory.Plan {
t.Helper()
plans, _ := inv.RecentPlans(ctx, 10)
for _, p := range plans {
if p.Release != nil {
return p
}
}
t.Fatal("no walk")
return inventory.Plan{}
}
// Judged until the first judging has found the node tools broken, and one more.
for i := 0; i < 20 && len(seen) < 2; i++ {
advancePlans(ctx, open)
}
if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
}
if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
t.Fatal("the node tools' build is not marked failed at once")
}
if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
}
if !slices.Contains(sends, "laptop:c1") {
t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
}
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
advancePlans(ctx, open)
if release().State != inventory.PlanRolling {
break
}
time.Sleep(20 * time.Millisecond)
}
p := release()
if p.State != inventory.PlanFailed {
t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
}
if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v; want its own pass", v)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
}
}
+41 -11
View File
@@ -492,17 +492,12 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
}
wait, waits := personWait(module, machine, h.Resources)
// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
// account already waiting before this send was not brought to it by this build, and a later build of
// the module sent while the login is still owed is judged as before. Two seconds of skew between the
// machine's clock and the controller's, as the witness allows.
if waits {
for _, r := range h.Resources {
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
waits = false
}
}
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
// what the controller sent, never from when the machine says the wait began — that time is the engine's
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
// have brought a wait, so a later build sent while the login is still owed is judged as before.
if waits && !f.groupsAdded[module] {
waits = false
}
for _, r := range h.Resources {
if r.Module != module {
@@ -565,3 +560,38 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
}
return out
}
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
// groups (ADR 0252).
func accountGroups(m catalogue.Manifest) map[string]bool {
out := map[string]bool{}
for _, r := range m.Resources {
if t, _ := r["type"].(string); t != "user" {
continue
}
name, _ := r["name"].(string)
groups, _ := r["groups"].([]any)
for _, g := range groups {
if group, ok := g.(string); ok && group != "" {
out[name+"/"+group] = true
}
}
}
return out
}
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
// the machine (no earlier manifest) adds every group it declares.
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
before := map[string]bool{}
if hadFrom {
before = accountGroups(from)
}
for g := range accountGroups(to) {
if !before[g] {
return true
}
}
return false
}
+41 -11
View File
@@ -6,6 +6,7 @@ import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
@@ -76,23 +77,19 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
now := time.Now()
since := now.Add(-time.Minute)
account := relogin("lights", "operator")
account.Since = since.Add(10 * time.Second) // the send put it in its wait
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}}
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}},
groupsAdded: map[string]bool{"lights": true}}
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
}
// **Only the send that put the account in its wait is excused** (issue 318 review): an account waiting
// since before the send is judged as before, so a later build of the module is not passed on an old wait.
older := f.health["laptop"]
older.Resources = []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}
older.Resources[0].Since = since.Add(-time.Hour)
f.health["laptop"] = older
// **Only a move that put the account in a new group is excused** (issue 318 review): a later build that
// adds no group did not bring the wait, and is judged as before.
f.groupsAdded["lights"] = false
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
t.Fatalf("a wait from before the send reads %v %q; want not yet", h, why)
t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why)
}
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}
f.groupsAdded["lights"] = true
h := f.health["laptop"]
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
@@ -273,6 +270,12 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
for _, e := range told.Said() {
if e.Change == conditions.ChangeCleared {
resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
// The clearing line is held to the plain rule too (ADR 0253).
w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation,
Resolved: e.Condition.Resolved, Needs: e.Condition.Needs}
if why, ok := conditions.PlainWords(w, "laptop"); !ok {
t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why)
}
}
}
}
@@ -280,3 +283,30 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
}
}
// Which moves add an account group, read from the builds' manifests (issue 318 review).
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
user := func(groups ...any) catalogue.Manifest {
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
"name": "operator", "groups": groups}}}
}
none := catalogue.Manifest{Module: "lights"}
for _, c := range []struct {
name string
from catalogue.Manifest
had bool
to catalogue.Manifest
addsSome bool
}{
{"a group added", none, true, user("lights"), true},
{"the same group kept", user("lights"), true, user("lights"), false},
{"a second group added", user("lights"), true, user("lights", "video"), true},
{"a group taken away", user("lights", "video"), true, user("lights"), false},
{"new to the machine, with a group", none, false, user("lights"), true},
{"new to the machine, no group", none, false, none, false},
} {
if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome {
t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome)
}
}
}
+65
View File
@@ -334,6 +334,9 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
notes = append(notes, p.Note)
}
done := map[string]bool{except: true}
for _, m := range g.Returned {
done[m] = true // put back at once when it broke
}
// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
// left without a verdict is one more move every other walk would wait for.
@@ -390,6 +393,65 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
return kept
}
// putBackBroken puts back, at once, every module a gate found broken and has not put back yet (issue 318
// review): a witness that reverted a core component, or a machine that refused or failed its send, is not
// left registered at the build that broke for as long as the modules beside it take to be judged — a resend
// meanwhile would declare it again. The send goes on judging the others to their own verdicts; its own
// verdict is still failed. lead and leadState are the module a plan's gate is kept on, when it is a plan's.
// Answers whether anything was put back.
func putBackBroken(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, lead string,
leadState *inventory.PlanModule) bool {
var todo []string
for _, m := range g.Broken {
if !slices.Contains(g.Returned, m) {
todo = append(todo, m)
}
}
if len(todo) == 0 || g.Verdict != "" {
return false
}
stateWas, noteWas := p.State, p.Note
batched, back := batchingRollbacks(ctx)
var said []string
for _, m := range todo {
g.Returned = append(g.Returned, m)
machines := g.Machines
var state *inventory.PlanModule
if m == lead && leadState != nil {
judged := *leadState
judged.Gate = nil // its own record of the failure; the send's gate goes on judging
state = &judged
} else {
i := slices.IndexFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == m })
if i < 0 {
continue
}
c := g.Carried[i]
state = &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
if sent, err := sentTheBuild(ctx, open, m, c.To); err == nil && len(sent) > 0 {
machines = sent
} else {
machines = []string{c.Node}
}
}
p.Note = ""
gateFailed(batched, open, p, m, state, machines, g.BrokenWhy)
if m == lead && leadState != nil {
leadState.Why = "put back at once, its send still judged: " + g.BrokenWhy
}
said = append(said, m)
}
sendRollbacks(ctx, open, p, back)
p.State = stateWas
p.Note = noteWas
if len(said) > 0 {
p.Note = fmt.Sprintf("put back at once: %s (%s); the rest of the send judged to their own verdicts",
strings.Join(said, ", "), g.BrokenWhy)
fmt.Printf("%s: %s\n", p.ID, p.Note)
}
return true
}
// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
func passedAloneWhy(g *inventory.PlanGate, module string) string {
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
@@ -617,6 +679,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
}
switch verdict {
case "":
if putBackBroken(ctx, open, p, g, "", nil) {
return true, nil
}
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
changed := p.Note != note
p.Note = note
+10 -1
View File
@@ -729,6 +729,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
}
switch verdict {
case "":
if putBackBroken(ctx, open, p, state.Gate, m, state) {
return true, nil
}
pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m,
strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate)))
continue
@@ -961,7 +964,13 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
fmt.Printf("%s: %s\n", p.ID, p.Note)
}()
g := state.Gate
if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
if g != nil && slices.Contains(g.Returned, module) {
// Put back at once when it broke: its rollback was made then, and is not made again.
state.Why = "put back when it broke; its send failed: " + g.Why
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was put back when it broke",
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
} else if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
gateFailed(batched, open, p, module, state, machines, why)
} else if slices.Contains(g.Passing, module) && state.Build != "" {
// **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes
+8 -2
View File
@@ -40,13 +40,19 @@ func TestReplay318(t *testing.T) {
t.Cleanup(func() { doctorFrom = was })
build := func(module, repository, commit string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
m := catalogue.Manifest{Module: module, Version: "1"}
if module == "openrazer" && commit == "c78b5fc9" {
// The build that put the operator's account in the group `openrazer` (ADR 0252).
m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator",
"groups": []any{"openrazer"}}}
}
manifest, _ := json.Marshal(m)
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
if err := inv.RegisterModule(ctx, m, inventory.Source{
Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
+2
View File
@@ -159,6 +159,8 @@ type PlanGate struct {
// first reason: the send fails, and the modules beside them are judged to their own verdict first.
Broken []string `json:"broken,omitempty"`
BrokenWhy string `json:"broken_why,omitempty"`
// Returned names the broken modules already put back, at once, while the rest of the send is judged.
Returned []string `json:"returned,omitempty"`
}
// CarriedMove is one module's build moving on a machine with a gated send.