diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index b2081d9d..68f3a24a 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -645,6 +645,10 @@ func doesWords(act conditions.Action) string { return "the delivery is stopped" case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "": return "the failed builds are asked again" + case act.Verb == broker.ControllerSeat+".push" && act.Arguments["node"] != "": + return "the held builds are sent to " + act.Arguments["node"] + case act.Verb == broker.ControllerSeat+".upgrade" && act.Arguments["release-backlog"] != "": + return "the builds held after the failed delivery are released" case strings.HasSuffix(act.Verb, ".restart"): return "its service is restarted on " + act.Machine } diff --git a/cmd/mesh-controller/held_moves.go b/cmd/mesh-controller/held_moves.go new file mode 100644 index 00000000..eaf63bc5 --- /dev/null +++ b/cmd/mesh-controller/held_moves.go @@ -0,0 +1,169 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). +// +// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that +// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and +// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on +// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way. +// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the +// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends +// when the machine no longer waits, whoever pushed it. +// +// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has +// its own row (S17). + +// kindMovesHeld is S24's kind: machine..moves-held. +const kindMovesHeld = "moves-held" + +// heldFacts is one machine and the moves of recorded modules that wait there for a person. +type heldFacts struct { + node string + moves []heldMove +} + +// gatherHeldMoves is, per machine, the moves that wait for a person and are no person's own choice, sorted by machine. +func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) { + nodes, err := inv.Nodes(ctx) + if err != nil { + return nil, fmt.Errorf("the machines cannot be read: %w", err) + } + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, err + } + var out []heldFacts + for _, n := range nodes { + moves, err := heldMovesOn(ctx, inv, f, n.Name) + if err != nil { + return nil, err + } + var asked []heldMove + for _, m := range moves { + if !m.Chosen { + asked = append(asked, m) + } + } + if len(asked) > 0 { + out = append(out, heldFacts{node: n.Name, moves: asked}) + } + } + return out, nil +} + +// modules are the held moves' modules, in order. +func (h heldFacts) modules() []string { + out := make([]string, 0, len(h.moves)) + for _, m := range h.moves { + out = append(out, m.Module) + } + return out +} + +// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push +// names every module whose move waits there as one that may move its data: the operator's Approve is the word +// ADR 0217 asks for, and the ask says so. +func (h heldFacts) releaseAction() conditions.Action { + return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove, + Arguments: map[string]string{"node": h.node, "move": strings.Join(h.modules(), ","), "why": "", + "cause": conditions.CauseOperatorAnswer}} +} + +// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does. +func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) { + names := namesWords(h.modules(), 3) + headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax) + recreates, data := false, false + for _, m := range h.moves { + switch { + case strings.Contains(m.Why, "recreates"): + recreates = true + default: + data = true + } + } + why := "would change what it runs" + switch { + case recreates && data: + why = "would restart a container or touch its data" + case recreates: + why = "would restart one of its containers" + case data: + why = "would touch its files or data" + } + explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ + "it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+ + "has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax) + needs = "release them " + FromMeshMCPServer + resolved = "Resolved: " + names + " is no longer waiting on " + h.node + return +} + +// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the +// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts. +func watchMovesHeld(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, h := range f.held { + var said []string + for _, m := range h.moves { + said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why)) + } + headline, explanation, needs, resolved := heldWords(h) + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld, + Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them", + h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node), + Said: strings.Join(said, "; "), + Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved, + Actions: []conditions.Action{h.releaseAction()}}) + } + return out +} + +// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a +// person: "module on machine", sorted. The plan's own record says the move waited when it was judged +// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older +// than the build now held is not named. +func planHeld(p inventory.Plan, held []heldFacts) []string { + var out []string + for _, h := range held { + for _, m := range h.moves { + s := p.Modules[m.Module] + if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) { + continue + } + out = append(out, m.Module+" on "+h.node) + } + } + sort.Strings(out) + return out +} + +// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a +// plan whose module waits for a push is not plainly "done". +func heldSaid(p inventory.Plan, held []heldFacts) string { + waiting := planHeld(p, held) + if len(waiting) == 0 { + return "" + } + return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", ")) +} + +// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before. +func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts { + held, err := gatherHeldMoves(ctx, inv) + if err != nil { + return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read + } + return held +} diff --git a/cmd/mesh-controller/held_moves_test.go b/cmd/mesh-controller/held_moves_test.go new file mode 100644 index 00000000..4923462c --- /dev/null +++ b/cmd/mesh-controller/held_moves_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). + +// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words. +func aHeldMailu() heldFacts { + return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb", + Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}} +} + +func heldConditionOn(f *signalFacts) conditions.Condition { + o := watchMovesHeld(f)[0] + return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} +} + +// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with +// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there. +func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + f.held = []heldFacts{aHeldMailu()} + got := watchMovesHeld(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning || + o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" { + t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key()) + } + if len(o.Actions) != 1 { + t.Fatalf("actions: %+v", o.Actions) + } + a := o.Actions[0] + if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove || + a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { + t.Fatalf("the action is not the push of novox: %+v", a) + } + if _, takes := a.Arguments["why"]; !takes { + t.Fatal("the push takes a why, which the asker fills in with the operator's answer") + } + for _, want := range []string{"mailu", "novox"} { + if !strings.Contains(o.Headline+o.Explanation, want) { + t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation) + } + } + if !strings.Contains(o.Said, "recreates 1 of mailu's 12") { + t.Errorf("the evidence does not carry the walk's own words: %q", o.Said) + } + if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax { + t.Errorf("headline: %q", o.Headline) + } +} + +// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent. +func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + if got := watchMovesHeld(f); len(got) != 0 { + t.Fatalf("%+v", got) + } +} + +// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the +// operator's warrant calls the controller's own push with the machine and the modules, once. +func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + f.held = []heldFacts{aHeldMailu()} + c := heldConditionOn(f) + r := newAskerRig(t) + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 1 || len(sent[0].Options) != 1 { + t.Fatalf("asked %+v", sent) + } + o := sent[0].Options[0] + if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" { + t.Errorf("the option is %+v", o) + } + if err := sent[0].Check(r.now); err != nil { + t.Errorf("the ask is refused as made: %v", err) + } + w := r.warrantFor(t, c.Key, "Release") + answerWith(t, r, w) + answerWith(t, r, w) + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") || + !strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) || + !strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) || + !strings.Contains(r.called[0], "chose Release") { + t.Fatalf("called %v", r.called) + } +} + +// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them. +func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) { + for _, verb := range []string{"push", "upgrade"} { + granted := false + for _, v := range broker.VerbsTheControllerActsOnAWarrant { + granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb) + } + if !granted { + t.Errorf("the controller may not act on a warrant for %s", verb) + } + } +} + +func aDoneMailuPlan() inventory.Plan { + return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone, + Tiers: [][]string{{"a"}, {"b"}, {"mailu"}}, + Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc", + Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}} +} + +// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine +// no longer holds it, or holds another build of the module, the line says nothing more. +func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) { + p := aDoneMailuPlan() + line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()}) + if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") { + t.Errorf("the line: %q", line) + } + if said := heldSaid(p, nil); said != "" { + t.Errorf("a plan whose moves were sent says %q", said) + } + other := aHeldMailu() + other.moves[0].To = "9999999ffff" + if said := heldSaid(p, []heldFacts{other}); said != "" { + t.Errorf("a plan older than the build now held says %q", said) + } + p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy} + if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" { + t.Errorf("a move the plan sent says %q", said) + } +} + +// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers +// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant. +func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) { + saved := backlogNow + t.Cleanup(func() { backlogNow = saved }) + backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person" + backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}} + got := backlogObservation() + if len(got) != 1 || len(got[0].Actions) != 1 { + t.Fatalf("raised %+v", got) + } + a := got[0].Actions[0] + if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove || + a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { + t.Fatalf("the action: %+v", a) + } + o := got[0] + c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} + r := newAskerRig(t) + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" { + t.Fatalf("asked %+v", sent) + } + w := r.warrantFor(t, c.Key, "Release") + answerWith(t, r, w) + answerWith(t, r, w) + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") || + !strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") { + t.Fatalf("called %v", r.called) + } +} + +// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for +// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is. +func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) { + saved := backlogNow + t.Cleanup(func() { backlogNow = saved }) + now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC) + p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding, + Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute), + Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}}, + Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))} + backlogNow.held = "" + if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") { + t.Errorf("the wait is not said: %q", line) + } + backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person" + line := planLine(p, now) + if strings.Contains(line, "LATE") || strings.Contains(line, "building") || + !strings.Contains(line, "until a person releases it") { + t.Errorf("a plan held behind a failed release reads %q", line) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 41ebf533..03e717a4 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -457,6 +457,11 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "A delivery did not finish. Nothing is tried again until you say so.", Resolved: "Resolved: the delivery is no longer failed"} }), + kindMovesHeld: worded(func(o conditions.Observation) words { + return words{Headline: "An update waits for you", + Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.", + Resolved: "Resolved: the update is no longer waiting"} + }), "ask-lost": worded(func(o conditions.Observation) words { return words{Headline: "A build has no result", Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go index 97a9dc5d..653eaf1c 100644 --- a/cmd/mesh-controller/plain_words_test.go +++ b/cmd/mesh-controller/plain_words_test.go @@ -260,7 +260,7 @@ func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) { plainExample(t, got[0], "Updates of openrazer and sensors wait for your release", "Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+ "sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+ - "They are not delivered until then, and stay held if you leave them.") + "They are not delivered until then, and stay held if you leave them.", "Release") } // **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any diff --git a/cmd/mesh-controller/plans_follow.go b/cmd/mesh-controller/plans_follow.go index cf48e194..cae2f5ba 100644 --- a/cmd/mesh-controller/plans_follow.go +++ b/cmd/mesh-controller/plans_follow.go @@ -153,7 +153,7 @@ func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsk line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string { on, _ := inv.Running(ctx, module) return on - }, waitsOf(ctx, inv, one)) + }, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv)) doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line) return json.NewEncoder(os.Stdout).Encode(doc) } diff --git a/cmd/mesh-controller/recorded_moves.go b/cmd/mesh-controller/recorded_moves.go index 156b7466..be6eb18f 100644 --- a/cmd/mesh-controller/recorded_moves.go +++ b/cmd/mesh-controller/recorded_moves.go @@ -181,15 +181,24 @@ func planModuleNames(p inventory.Plan) map[string]string { return out } -// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why -// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a -// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. -func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { +// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6). +type heldMove struct { + Module string + From, To string + // Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold. + Why string + // Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the + // operator is not asked about again: they chose that hold, and the bus has its own condition (S17). + Chosen bool +} + +// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order. +func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) { sent, known, err := inv.SentBuilds(ctx, node) if err != nil || !known { return nil, err } - var out []string + var out []heldMove for _, m := range sortedKeysOf(sent) { was := sent[m] now, held := f.current[m] @@ -197,12 +206,25 @@ func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, continue } if v := f.recordedMove(m, was); !v.rolls { - out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said)) + out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said, + Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus}) } } return out, nil } +// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why +// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a +// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. +func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { + moves, err := heldMovesOn(ctx, inv, f, node) + var out []string + for _, h := range moves { + out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why)) + } + return out, err +} + // stringsKeys is a map of lists as a map of its keys, for sortedKeysOf. func stringsKeys(m map[string][]string) map[string]string { out := map[string]string{} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index ff063b71..348468c8 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -1371,6 +1371,10 @@ func backlogObservation() []conditions.Observation { "them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)} w := releaseHeldWords(modules, machines) o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs + // Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own + // `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged. + o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove, + Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}} return []conditions.Observation{o} } diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 430cb335..85984ec6 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -2061,6 +2061,12 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D // refused ("a build waits there for its gate: (...)") is kept there. "building" with LATE // hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920). what = p.Note + // And when the builds are held after a failed release, they wait for a person, not for a build, and the + // operator is asked to release them (novox/hq issue 1732): said so, and not late. + if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" { + what += " — held after a failed delivery until a person releases it (the operator is asked)" + late = "" + } return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late) } return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late) @@ -2268,7 +2274,7 @@ func plansCommand(ctx context.Context, args []string) error { bounds.of(p.Repository), func(module string) []string { on, _ := inv.Running(ctx, module) return on - }, waitsOf(ctx, inv, []inventory.Plan{p}))) + }, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv))) if r := p.Release; r != nil { // A release plan's walk (ADR 0236): machines done, the one judged, those to come. fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), @@ -2463,11 +2469,12 @@ func plansCommand(ctx context.Context, args []string) error { return on } waits := waitsOf(ctx, inv, plans) + held := heldNow(ctx, inv) for _, p := range plans { if p.Batch() { continue } - fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits)) + fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits), heldSaid(p, held)) } return nil } diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 33a49603..f1e02351 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -244,6 +244,14 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since }) }}, + {Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator", + Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)", + Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " + + "machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " + + "pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.", + Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld, + newest: func(f *signalFacts) time.Time { return time.Time{} }}, {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 488117da..d583d35a 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -171,6 +171,12 @@ var suppressions = map[string]suppression{ since: f.now.Add(-time.Second)}} }, }, + // A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732). + // Inside: nothing waits there. + "S24": { + inside: func(f *signalFacts) { f.held = nil }, + past: func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} }, + }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index a5f39a8a..462a80e1 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -58,6 +58,9 @@ type signalFacts struct { // failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449). failed []failedPlanFacts failedErr error + // held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732). + held []heldFacts + heldErr error // refusedSends are the sends a machine refused within the hour for the generation they were composed // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send @@ -357,6 +360,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { f.batches, f.plansErr = gatherBatches(ctx, inv, now) } f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) + f.held, f.heldErr = gatherHeldMoves(ctx, inv) f.loop, f.loopErr = w.gatherLoop() f.asks, f.asksErr = w.gatherAsks(ctx, inv) if open, err := w.keeper.Open(ctx); err != nil { diff --git a/internal/broker/nats.go b/internal/broker/nats.go index c9007a14..e6ff4265 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -361,9 +361,11 @@ var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"} // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the -// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat. +// controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release +// sent by its own `upgrade release-backlog` (novox/hq issue 1732). Named one by one; a node seat's on any machine, a +// mesh seat's flat. var VerbsTheControllerActsOnAWarrant = []SeatVerb{{Seat: "node-service-manager", Verb: "restart"}, - {Seat: ControllerSeat, Verb: "plans"}} + {Seat: ControllerSeat, Verb: "plans"}, {Seat: ControllerSeat, Verb: "push"}, {Seat: ControllerSeat, Verb: "upgrade"}} // VerbsTheControllerAsksTheOperatorChannel are the operator channel's verbs the controller calls (novox/hq // ADR 0380): `tell`, one plain message with no condition behind it — a module's own secret given, named, @@ -387,6 +389,16 @@ type DegradableGrant struct { // works. Every other grant stays required — missing one of those is a controller that comes up and is // refused on the first thing it tries, with no fallback that covers it. var DegradableGrants = []DegradableGrant{ + { + Subject: "mesh.seat." + ControllerSeat + ".tool.push", + Reason: "refused: the ask of a held move (Release) is answered, and the refusal told, while the operator's own " + + "push at the terminal still sends the machine (issue 1732; ADR 0259)", + }, + { + Subject: "mesh.seat." + ControllerSeat + ".tool.upgrade", + Reason: "refused: the ask of the builds held after a failed release is answered, and the refusal told, while the " + + "operator's own `upgrade release-backlog` at the terminal still releases them (issue 1732; ADR 0259)", + }, { Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb, Reason: "refused: the controller raises the secret-given condition as before (ADR 0380 §9)", diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 1e2b427d..835d59c9 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -29,7 +29,7 @@ accounts { ] users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } }