From e3b5c224e85655c7c4c0b87d87e0622ec66d45a6 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 13:41:47 +0200 Subject: [PATCH] Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318) --- cmd/mesh-controller/gate.go | 99 +++++++++- cmd/mesh-controller/module_health.go | 102 +++++++++- cmd/mesh-controller/person_wait_test.go | 195 +++++++++++++++++++ cmd/mesh-controller/release.go | 37 ++++ cmd/mesh-controller/replay318_test.go | 244 ++++++++++++++++++++++++ internal/inventory/health.go | 3 + internal/inventory/plans.go | 9 + internal/link/protocol.go | 13 ++ 8 files changed, 694 insertions(+), 8 deletions(-) create mode 100644 cmd/mesh-controller/person_wait_test.go create mode 100644 cmd/mesh-controller/replay318_test.go diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 7f764557..e821b716 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -77,6 +77,10 @@ type health int const ( healthGood health = iota + // healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for + // one person's new login (personWait). The build did what it should; no bound a machine can meet ends + // the wait. It counts as a pass, and the gate carries the reading along in its verdict. + healthPerson // healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy // provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a // build for something it did not do. @@ -209,7 +213,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string, firstLine(f.openErr.Error()) } for _, c := range f.open { - if c.Source == gateProbe || c.Raised.Before(since) { + // A wait for a person's new login is the module's reading, not a fault raised since the send: the + // gate reads it from the statement below (ADR 0254). + if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded { continue } onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || @@ -435,13 +441,19 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs for node, moved := range byMachine { words[node] = aboutTheMachine(node, moved, *g.Since, facts) } + // **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its + // machines', its passes are counted apart, and the send's verdict is still one — but a module that was + // healthy on its own for the passes the gate asks keeps a pass when another beside it fails. worst, why := healthGood, "" var failing []string broken := map[string]bool{} + reading := map[string]health{} + waits := map[string]string{} + var modules []string for _, j := range pairs { w := words[j.node] h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts) - if h == healthGood { + if h == healthGood || h == healthPerson { if on, named := w.on[j.module]; named { h, said = healthNotYet, on } else if w.whole != "" { @@ -450,7 +462,16 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs h, said = healthWaiting, w.waiting } } - if h != healthGood && !slices.Contains(failing, j.module) { + if _, seen := reading[j.module]; !seen { + modules = append(modules, j.module) + } + if h > reading[j.module] { + reading[j.module] = h + } + if h == healthPerson { + waits[j.module] = joinSaid(waits[j.module], said) + } + if h > healthPerson && !slices.Contains(failing, j.module) { failing = append(failing, j.module) } if h == healthBroken { @@ -458,14 +479,45 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs } if h > worst { worst, why = h, said - } else if h == worst && h != healthGood && why == "" { + } else if h == worst && h > healthPerson && why == "" { why = said } } + if g.Healthy == nil { + g.Healthy = map[string]int{} + } + g.Waits = nil + for _, m := range modules { + switch { + case reading[m] > healthPerson: + g.Healthy[m] = 0 + default: + g.Healthy[m]++ + if w := waits[m]; w != "" { + if g.Waits == nil { + g.Waits = map[string]string{} + } + g.Waits[m] = w + } + } + } + // passedAlone is every module that passed on its own while the send as a whole did not. + passedAlone := func() []string { + var out []string + if now.Sub(*g.Since) < gateSettle { + return nil + } + for _, m := range modules { + if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses { + out = append(out, m) + } + } + return out + } switch { case worst == healthBroken: // What broke is put back; what was only not yet healthy beside it is too — they moved together. - g.Failing = failing + g.Failing, g.Passing = failing, passedAlone() decide(g, inventory.GateFailed, why, now) case worst == healthWaiting: // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — @@ -474,19 +526,48 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs case worst == healthNotYet: g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing if now.Sub(*g.Since) > gateBound { + g.Passing = passedAlone() decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now) } default: + // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. g.Passes++ g.LastPass, g.Last, g.Failing = &now, "", nil if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, - now.Sub(*g.Since).Round(time.Second)), now) + now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now) } } return g.Verdict, nil } +// waitsSaid is the waits for a person a passing gate carries, as its verdict says them. +func waitsSaid(waits map[string]string) string { + if len(waits) == 0 { + return "" + } + modules := make([]string, 0, len(waits)) + for m := range waits { + modules = append(modules, m) + } + sort.Strings(modules) + var said []string + for _, m := range modules { + said = append(said, waits[m]) + } + return "; and it waits for a person: " + strings.Join(said, "; ") +} + +func joinSaid(a, b string) string { + switch { + case a == "": + return b + case b == "" || strings.Contains(a, b): + return a + } + return a + "; " + b +} + // decide sets a gate's verdict. func decide(g *inventory.PlanGate, verdict, why string, now time.Time) { g.Verdict, g.Why, g.JudgedAt = verdict, why, &now @@ -959,6 +1040,12 @@ func gateLine(g *inventory.PlanGate) string { } else if g.Verdict == "" { line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses) } + if g.Verdict == "" && len(g.Waits) > 0 { + line += waitsSaid(g.Waits) + } + if len(g.Passing) > 0 { + line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ") + } if g.Rollback != "" { line += "; " + g.Rollback } diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index d402c94a..0d6ddc15 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -73,7 +73,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke for _, r := range h.Resources { kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts, - Check: r.Check, Needs: r.Needs} + Check: r.Check, Needs: r.Needs, Account: r.Account} resources = append(resources, kept) if r.State == link.StateUnhealthy && r.Module != "" { unhealthy[r.Module] = append(unhealthy[r.Module], kept) @@ -135,7 +135,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } standing := map[string]conditions.Condition{} for _, c := range open { - if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node { + if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded) && c.Subject.Machine == node { standing[c.Key] = c } } @@ -155,6 +155,19 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi heldOn := map[string]string{} providers := map[catalogue.Chosen]bool{} for _, m := range modules { + // **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the + // operator, never urgent, cleared on the first statement that no longer says it. + if said, waits := personWait(m, node, unhealthy[m]); waits { + o := reloginObservation(m, node, said, unhealthy[m]) + seen[o.Key()] = true + if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen { + continue + } + if _, err := k.Observe(ctx, o); err != nil { + problems = append(problems, err.Error()) + } + continue + } o := moduleUnhealthyObservation(m, node, unhealthy[m]) if hold != nil { if p, held := hold.heldUnder(node, m, unhealthy[m]); held { @@ -188,6 +201,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi } module := strings.TrimSuffix(c.Subject.ID, "."+node) why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module) + if c.Kind == kindReloginNeeded { + why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module) + } if on, held := heldOn[key]; held { why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on) } @@ -239,6 +255,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata return err } +// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the +// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. +func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation { + o := moduleUnhealthyObservation(module, node, rs) + o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said + return o +} + // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module, // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence. func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { @@ -296,9 +320,76 @@ func orNotSaid(s string) string { return s } +// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq +// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault. +const kindReloginNeeded = "relogin-needed" + +// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and +// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never +// excused: +// +// - at least one of the module's resources is its account (kind account), unhealthy with a reason that +// starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session +// running began before; +// - every other unhealthy resource of the module runs in the own service manager of one of those very +// accounts (Account names it): the manager that began before the group and does not hold it. +// +// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in +// another account's manager, a resource whose engine does not say whose manager it is in, an account +// not in its group or that could not be read — is not a wait, and the module is judged as before. A +// resource still starting is not unhealthy and does not make a wait either. Pure. +func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) { + waiting := map[string]bool{} + var accounts []string + for _, r := range rs { + if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy && + strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] { + waiting[accountOf(r)] = true + accounts = append(accounts, accountOf(r)) + } + } + if len(accounts) == 0 { + return "", false + } + var units []string + for _, r := range rs { + if r.Module != module || r.State != link.StateUnhealthy { + continue + } + switch { + case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]: + case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]: + units = append(units, r.Target) + default: + return "", false + } + } + sort.Strings(accounts) + said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+ + "running session began before it was; log out of every session and in again, or reboot", machine, module, + strings.Join(accounts, ", ")) + if len(units) > 0 { + sort.Strings(units) + said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", ")) + } + return said, true +} + +// accountOf is the account a resource of kind account is: named by the engine, or its target. +func accountOf(r inventory.ResourceHealth) string { + if r.Account != "" { + return r.Account + } + return r.Target +} + // moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236 // §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since // the send; not yet otherwise, saying which. A machine that never stated health is judged as before. +// +// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module +// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build, +// and not something a machine can meet within a bound. func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) { if f.healthErr != nil { return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error()) @@ -310,10 +401,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea if h.HeardAt.Before(since) { return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module) } + wait, waits := personWait(module, machine, h.Resources) for _, r := range h.Resources { if r.Module != module { continue } + if waits && r.State == link.StateUnhealthy { + continue + } switch r.State { case link.StateHealthy: case link.StateStarting: @@ -329,6 +424,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea reasonAfter(r.Reason)) } } + if waits { + return healthPerson, wait + } return healthGood, "" } diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go new file mode 100644 index 00000000..3030f80b --- /dev/null +++ b/cmd/mesh-controller/person_wait_test.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254, +// issue 318). + +func relogin(module, account string) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account, + Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module + + ", and its running session began before it was; log out of every session and in again, or reboot"} +} + +func userUnit(module, account string) inventory.ResourceHealth { + return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit, + Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy, + Reason: "failed in the account's own service manager (exit-code)"} +} + +// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as +// before, so that a fault is never excused. +func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { + notInGroup := relogin("lights", "operator") + notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" + systemUnit := userUnit("lights", "operator") + systemUnit.Account = "" + otherAccount := userUnit("lights", "guest") + container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights", + State: link.StateUnhealthy, Reason: "down"} + otherModule := userUnit("sound", "operator") + starting := userUnit("lights", "operator") + starting.State = link.StateStarting + byTarget := relogin("lights", "operator") + byTarget.Account = "" // an account said by its target alone is still that account + + for _, c := range []struct { + name string + rs []inventory.ResourceHealth + waits bool + }{ + {"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true}, + {"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), + userUnit("lights", "operator")}, true}, + {"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true}, + {"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true}, + {"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true}, + {"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false}, + {"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false}, + {"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false}, + {"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false}, + {"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false}, + } { + said, waits := personWait("lights", "laptop", c.rs) + if waits != c.waits { + t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits) + continue + } + if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") { + t.Errorf("%s: said %q", c.name, said) + } + } +} + +// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module +// with a container down beside it is not yet healthy, as before. +func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { + now := time.Now() + since := now.Add(-time.Minute) + f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, + Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}} + if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") { + t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why) + } + h := f.health["laptop"] + h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", + Target: "lights", State: link.StateUnhealthy, Reason: "down"}) + f.health["laptop"] = h + if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet { + t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why) + } +} + +// The module's condition says the wait in one sentence for a person, and clears on the first statement that +// no longer says it — said as the module's own fault when its unit still fails after the new login. +func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + at := h0 + say := func(rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, + Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + asLink := func(r inventory.ResourceHealth) link.ResourceHealth { + return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, + Reason: r.Reason, Account: r.Account} + } + account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) + openNow := func() []conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + say(account, unit) + say(account, unit) + list := openNow() + if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { + t.Fatalf("raised %+v; want lights' relogin-needed alone", list) + } + c := list[0] + if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || + !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { + t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) + } + // After the new login the account is healthy, and the unit, still failing, is the module's own fault. + account.State, account.Reason = link.StateHealthy, "" + say(account, unit) + list = openNow() + if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { + t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) + } + unit.State, unit.Reason = link.StateHealthy, "" + say(account, unit) + if list = openNow(); len(list) != 0 { + t.Fatalf("after the unit runs: %+v; want nothing open", list) + } +} + +// A module that is healthy on its own keeps its pass when another module of the same send fails at the +// bound: it is neither put back nor left without a verdict for every other walk to wait on. +func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"laptop": true}, nil + } + backlogFacts := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := backlogFacts(ctx, open, component) + f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), + Resources: []inventory.ResourceHealth{ + {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, + {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, + Reason: "down"}}}} + return f, err + } + gateEvery, gateBound = 0, 300*time.Millisecond + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + advancePlans(ctx, b.open) + if p := b.release(t); p.State != inventory.PlanRolling { + break + } + time.Sleep(20 * time.Millisecond) + } + p := b.release(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) + } + if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || + !strings.Contains(v.Why, "on its own") { + t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { + t.Fatal("late's build is not marked failed at its gate") + } + current, _ := inv.CurrentBuilds(ctx) + if current["app"].Commit != "c2" || current["late"].Commit != "c1" { + t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, + current["late"].Commit) + } + if !strings.Contains(p.Note, "passed on their own and kept: app") { + t.Errorf("the walk does not say what kept its pass: %s", p.Note) + } +} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 97f30df4..ef480603 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -334,6 +334,15 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent notes = append(notes, p.Note) } done := map[string]bool{except: true} + // **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the + // gate asks while another module of the send failed, it is neither put back nor left unjudged — a build + // left without a verdict is one more move every other walk would wait for. + if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 { + notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", ")) + for _, m := range kept { + done[m] = true + } + } for _, c := range g.Carried { if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) { continue @@ -356,6 +365,34 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent p.Note = strings.Join(notes, "; ") } +// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the +// send failed (ADR 0254), except the plan's own module; answers the modules kept. +func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string { + var kept []string + seen := map[string]bool{} + for _, c := range g.Carried { + if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) { + continue + } + seen[c.Build] = true + why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why) + if w := g.Waits[c.Module]; w != "" { + why += "; and it waits for a person: " + w + } + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, + Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why, + Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { + fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module, + c.Node, err) + continue + } + if !slices.Contains(kept, c.Module) { + kept = append(kept, c.Module) + } + } + return kept +} + // sentTheBuild is every machine running a module that was last sent this build of it. func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) { running, err := open.inventory.Running(ctx, module) diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go new file mode 100644 index 00000000..d829a5e7 --- /dev/null +++ b/cmd/mesh-controller/replay318_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the +// older commit: the statements are read from the node-engine's JSON, as the controller hears them. +// +// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send — +// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged +// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the +// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service +// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its +// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and +// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there. +// +// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along, +// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence +// for the operator, that a new login is needed on the laptop. +func TestReplay318(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + keeper, _ := withConditionsInMemory(t) + was := doctorFrom + doctorFrom = nil + t.Cleanup(func() { doctorFrom = was }) + + build := func(module, repository, commit string, asked time.Time) { + manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, + Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ + Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, + Asked: asked}); err != nil { + t.Fatal(err) + } + } + type move struct{ module, repository, from, to string } + moves := []move{ + {"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"}, + {"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"}, + {"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"}, + } + machines := []string{"laptop", "workstation"} + if _, err := inv.AddNode(ctx, "workstation"); err != nil { + t.Fatal(err) + } + old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) + for _, mv := range moves { + build(mv.module, mv.repository, mv.from, old) + for _, n := range machines { + if _, err := inv.Assign(ctx, n, mv.module); err != nil { + t.Fatal(err) + } + } + } + for _, n := range machines { + sent := map[string]string{} + for _, mv := range moves { + sent[mv.module] = mv.from + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { + t.Fatal(err) + } + } + for _, mv := range moves { + build(mv.module, mv.repository, mv.to, recent) + } + + // What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the + // daemon's unit failed in that account's own manager; docker's container is healthy. + statement := func(at time.Time) link.Health { + raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [ + {"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service", + "state": "healthy", "since": %q}, + {"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator", + "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, + "reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"}, + {"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service", + "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, + "reason": "failed in the account's own service manager (exit-code)"}]}`, + link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), + at.Format(time.RFC3339Nano)) + var h link.Health + if err := json.Unmarshal([]byte(raw), &h); err != nil { + t.Fatal(err) + } + return h + } + + wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + t.Cleanup(func() { + machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + }) + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"laptop": true, "workstation": true}, nil + } + var sends [][]string + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + sends = append(sends, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + modules, _ := open.inventory.Assigned(ctx, node) + carried := map[string]string{} + for _, m := range modules { + carried[m] = current[m].Commit + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + // Every judging hears each machine's newest statement first, as the controller does between judgings: + // the account's wait and the failed unit are raised as openrazer's condition, after the send. + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + now := time.Now() + f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{}, + judged: true} + for _, m := range machines { + if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil { + return f, err + } + f.served[m] = served{runtime: true, tools: map[string]bool{}} + } + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + f.reports[r.Node] = r + } + if f.health, err = open.inventory.Healths(ctx); err != nil { + return f, err + } + f.open, f.openErr = keeper.Open(ctx) + return f, nil + } + // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. + gateSettle, gateEvery, gateBound = 0, 0, 0 + + for i := 0; i < 12; i++ { + advancePlans(ctx, open) + time.Sleep(5 * time.Millisecond) + } + + var p inventory.Plan + plans, err := inv.RecentPlans(ctx, 10) + if err != nil { + t.Fatal(err) + } + for _, q := range plans { + if q.Release != nil { + p = q + break + } + } + if p.Release == nil { + t.Fatal("no walk of the builds waiting for a gate was opened") + } + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) { + t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State, + p.Release.Done, p.Note) + } + if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) { + t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + if current["openrazer"].Commit != "c78b5fc9" { + t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit) + } + for _, mv := range moves { + v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to) + if err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err) + } + if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") { + t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why) + } + } + // What the operator reads: one condition per machine for openrazer, saying a new login is needed there, + // a warning for a person, and no fault of the build. + list, err := keeper.Open(ctx) + if err != nil { + t.Fatal(err) + } + var said []string + for _, c := range list { + said = append(said, c.Key+": "+c.Summary) + if c.Subject.ID == "openrazer.laptop" { + if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent || + c.Resolver != conditions.ResolverOperator { + t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary) + } + } + } + if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") { + t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n")) + } +} diff --git a/internal/inventory/health.go b/internal/inventory/health.go index 8fca6b8a..3272fd74 100644 --- a/internal/inventory/health.go +++ b/internal/inventory/health.go @@ -28,6 +28,9 @@ type ResourceHealth struct { // (ADR 0240 Phase B, to-be 48 §6). Check string `json:"check,omitempty"` Needs string `json:"needs,omitempty"` + // Account is the account whose own service manager runs it, or the account a resource of kind account + // is (novox/hq ADR 0254). + Account string `json:"account,omitempty"` } // NodeHealth is a machine's newest statement, as kept. diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index a86bf512..70b9d253 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -143,6 +143,15 @@ type PlanGate struct { Carried []CarriedMove `json:"carried,omitempty"` // Failing names the modules the last judging found wanting. Failing []string `json:"failing,omitempty"` + // Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged, + // or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's. + Healthy map[string]int `json:"healthy,omitempty"` + // Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never + // a reason to fail. + Waits map[string]string `json:"waits,omitempty"` + // Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its + // pass, and is not put back. + Passing []string `json:"passing,omitempty"` } // CarriedMove is one module's build moving on a machine with a gated send. diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 5d883a92..3b28d86c 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -441,6 +441,15 @@ type Health struct { // file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed. const KindUnit = "unit" +// KindAccount is an account a module puts in a group (novox/hq ADR 0252): healthy when the account's running +// session has every group the module declares, or nobody is logged in; unhealthy otherwise, its reason +// starting ReasonRelogin when only a new login is missing. +const KindAccount = "account" + +// ReasonRelogin starts the reason of an account whose running session began before it was put in a group +// (ADR 0252): the build did what it should, and a person has one step left (novox/hq ADR 0254). +const ReasonRelogin = "relogin needed" + // The states of a machine's service managers (issue 315). const ( UnitsRunning = "running" @@ -529,6 +538,10 @@ type ResourceHealth struct { // alone; Needs is the provision the check exercises (to-be 48 §6). Check string `json:"check,omitempty"` Needs string `json:"needs,omitempty"` + // Account is the account whose own service manager runs it, for a unit or service in an account's + // manager, and the account itself for a resource of kind KindAccount (novox/hq ADR 0254). Empty from an + // engine older than that, and for anything the machine's own manager or runtime runs. + Account string `json:"account,omitempty"` } // HealthSaid is the health event's body: the machine and its statement. The machine is read from the