diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index b2081d9d..68f3a24a 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -645,6 +645,10 @@ func doesWords(act conditions.Action) string { return "the delivery is stopped" case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "": return "the failed builds are asked again" + case act.Verb == broker.ControllerSeat+".push" && act.Arguments["node"] != "": + return "the held builds are sent to " + act.Arguments["node"] + case act.Verb == broker.ControllerSeat+".upgrade" && act.Arguments["release-backlog"] != "": + return "the builds held after the failed delivery are released" case strings.HasSuffix(act.Verb, ".restart"): return "its service is restarted on " + act.Machine } diff --git a/cmd/mesh-controller/compose_sdk.go b/cmd/mesh-controller/compose_sdk.go index 8eb0b66f..0a8f6b5e 100644 --- a/cmd/mesh-controller/compose_sdk.go +++ b/cmd/mesh-controller/compose_sdk.go @@ -239,7 +239,11 @@ func markUnresolvable(err error) error { // still names anything foreseen is refused, said, and never sent. func composeSend(ctx context.Context, open *stores, node string) (sendable, error) { declared, _, err := composeSendValidated(ctx, open, node) - return declared, err + if err != nil { + return declared, err + } + // Never a declaration naming an image the store does not hold (novox/hq issue 1742): that module is left out. + return withoutMissingImages(ctx, open, node, declared), nil } // composeSendValidated is composeSend and what the node-engine's validator refuses of the declaration it composed, for the diff --git a/cmd/mesh-controller/held_moves.go b/cmd/mesh-controller/held_moves.go new file mode 100644 index 00000000..ce3fe0e6 --- /dev/null +++ b/cmd/mesh-controller/held_moves.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). +// +// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that +// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and +// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on +// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way. +// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the +// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends +// when the machine no longer waits, whoever pushed it. +// +// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has +// its own row (S17). + +// kindMovesHeld is S24's kind: machine..moves-held. +const kindMovesHeld = "moves-held" + +// heldFacts is one machine and the moves of recorded modules that wait there for a person. +type heldFacts struct { + node string + moves []heldMove +} + +// gatherHeldMoves is, per machine that holds at least one move no person chose to hold, all its held moves, by machine. +func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) { + nodes, err := inv.Nodes(ctx) + if err != nil { + return nil, fmt.Errorf("the machines cannot be read: %w", err) + } + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, err + } + var out []heldFacts + for _, n := range nodes { + moves, err := heldMovesOn(ctx, inv, f, n.Name) + if err != nil { + return nil, err + } + h := heldFacts{node: n.Name, moves: moves} + if len(h.asked()) > 0 { + out = append(out, h) + } + } + return out, nil +} + +// asked are the moves no person chose to hold, and chosen the ones a person did (a module set to record with a why, the bus). +func (h heldFacts) asked() []heldMove { return h.byChoice(false) } +func (h heldFacts) chosen() []heldMove { return h.byChoice(true) } + +func (h heldFacts) byChoice(chosen bool) []heldMove { + var out []heldMove + for _, m := range h.moves { + if m.Chosen == chosen { + out = append(out, m) + } + } + return out +} + +// modules are the held moves' modules, in order, of all of them. +func (h heldFacts) modules() []string { return moduleNames(h.moves) } + +func moduleNames(moves []heldMove) []string { + out := make([]string, 0, len(moves)) + for _, m := range moves { + out = append(out, m.Module) + } + return out +} + +// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push +// names every module whose move waits there as one that may move its data: the operator's Approve is the word +// ADR 0217 asks for, and the ask says so. Offered only where no person's own hold is on the machine too: a push +// composes every recorded module at its current build, so it would send that move as well, which the ask never named +// (ADR 0259 §6), and hold the whole machine if that module's data would move. +func (h heldFacts) releaseAction() conditions.Action { + return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove, + Arguments: map[string]string{"node": h.node, "move": strings.Join(moduleNames(h.asked()), ","), "why": "", + "cause": conditions.CauseOperatorAnswer}} +} + +// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does. +func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) { + names := namesWords(moduleNames(h.asked()), 3) + headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax) + recreates, data := false, false + for _, m := range h.asked() { + switch { + case strings.Contains(m.Why, "recreates"): + recreates = true + default: + data = true + } + } + why := "would change what it runs" + switch { + case recreates && data: + why = "would restart a container or touch its data" + case recreates: + why = "would restart one of its containers" + case data: + why = "would touch its files or data" + } + explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ + "it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+ + "has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax) + needs = "release them " + FromMeshMCPServer + if chosen := h.chosen(); len(chosen) > 0 { + // A push of the machine would send the move a person chose to hold as well: no answer is offered here. + explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ + "it waits for you. %s is also held there by your own choice, and a push of %s would send that too, so "+ + "nothing is offered here: the push is yours at the controller's terminal.", + names, why, h.node, conditions.Capital(namesWords(moduleNames(chosen), 3)), h.node), conditions.ExplanationMax) + needs = fmt.Sprintf("push %s at the controller's terminal, once you have decided about %s.", h.node, + namesWords(moduleNames(chosen), 3)) + } + resolved = "Resolved: " + names + " is no longer waiting on " + h.node + return +} + +// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the +// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts. +func watchMovesHeld(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, h := range f.held { + var said []string + for _, m := range h.moves { + said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why)) + } + headline, explanation, needs, resolved := heldWords(h) + if c := h.chosen(); len(c) > 0 { + said = append(said, "held by a person's own choice: "+strings.Join(moduleNames(c), ", ")) + } + var actions []conditions.Action + if len(h.chosen()) == 0 { + actions = []conditions.Action{h.releaseAction()} + } + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld, + Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them", + h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node), + Said: strings.Join(said, "; "), + Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved, + Actions: actions}) + } + return out +} + +// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a +// person: "module on machine", sorted. The plan's own record says the move waited when it was judged +// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older +// than the build now held is not named. +func planHeld(p inventory.Plan, held []heldFacts) []string { + var out []string + for _, h := range held { + for _, m := range h.moves { + s := p.Modules[m.Module] + if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) { + continue + } + out = append(out, m.Module+" on "+h.node) + } + } + sort.Strings(out) + return out +} + +// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a +// plan whose module waits for a push is not plainly "done". +func heldSaid(p inventory.Plan, held []heldFacts) string { + waiting := planHeld(p, held) + if len(waiting) == 0 { + return "" + } + return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", ")) +} + +// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before. +func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts { + held, err := gatherHeldMoves(ctx, inv) + if err != nil { + return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read + } + return held +} diff --git a/cmd/mesh-controller/held_moves_integration_test.go b/cmd/mesh-controller/held_moves_integration_test.go new file mode 100644 index 00000000..55b65916 --- /dev/null +++ b/cmd/mesh-controller/held_moves_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} } + +func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts { + t.Helper() + f, err := readMoveFacts(t.Context(), rm.open.inventory) + if err != nil { + t.Fatal(err) + } + return f +} + +// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read +// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine +// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it. +func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) { + rm := aRecordedMesh(t, aMailu(true)) + c2 := aMailu(true) + resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + ctx := t.Context() + held, err := gatherHeldMoves(ctx, rm.open.inventory) + if err != nil { + t.Fatal(err) + } + if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 || + held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") { + t.Fatalf("held %+v", held) + } + if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil { + t.Fatal(err) + } + held, err = gatherHeldMoves(ctx, rm.open.inventory) + if err != nil || len(held) != 1 || held[0].node != "laptop" { + t.Fatalf("after anchor was sent c2: %+v (%v)", held, err) + } +} + +// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold. +func TestAHoldAPersonChoseIsNotAsked(t *testing.T) { + rm := aRecordedMesh(t, aMailu(false)) + c2 := aMailu(false) + c2.Bundles[0].Digest = "sha256:bbb" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil { + t.Fatal(err) + } + held, err := gatherHeldMoves(t.Context(), rm.open.inventory) + if err != nil || len(held) != 0 { + t.Fatalf("a machine holding only a person's own hold is asked: %+v (%v)", held, err) + } + if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen { + t.Fatalf("status still names it, marked as chosen: %+v", moves) + } +} + +// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing +// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice. +func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) { + m := aMailu(true) + resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64) + rm := aRecordedMesh(t, m) + asked := 0 + lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil } + imagesHeld.Lock() + imagesHeld.held = map[string]bool{} + imagesHeld.Unlock() + got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking) + if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" { + t.Fatalf("missing %+v (%v)", got, err) + } + holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil } + if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 { + t.Fatalf("an image the store holds is missing: %+v (%v)", got, err) + } + before := asked + if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before { + t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err) + } +} diff --git a/cmd/mesh-controller/held_moves_test.go b/cmd/mesh-controller/held_moves_test.go new file mode 100644 index 00000000..759837d7 --- /dev/null +++ b/cmd/mesh-controller/held_moves_test.go @@ -0,0 +1,237 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). + +// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words. +func aHeldMailu() heldFacts { + return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb", + Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}} +} + +func heldConditionOn(f *signalFacts) conditions.Condition { + o := watchMovesHeld(f)[0] + return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} +} + +// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with +// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there. +func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + f.held = []heldFacts{aHeldMailu()} + got := watchMovesHeld(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning || + o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" { + t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key()) + } + if len(o.Actions) != 1 { + t.Fatalf("actions: %+v", o.Actions) + } + a := o.Actions[0] + if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove || + a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { + t.Fatalf("the action is not the push of novox: %+v", a) + } + if _, takes := a.Arguments["why"]; !takes { + t.Fatal("the push takes a why, which the asker fills in with the operator's answer") + } + for _, want := range []string{"mailu", "novox"} { + if !strings.Contains(o.Headline+o.Explanation, want) { + t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation) + } + } + if !strings.Contains(o.Said, "recreates 1 of mailu's 12") { + t.Errorf("the evidence does not carry the walk's own words: %q", o.Said) + } + if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax { + t.Errorf("headline: %q", o.Headline) + } +} + +// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent. +func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + if got := watchMovesHeld(f); len(got) != 0 { + t.Fatalf("%+v", got) + } +} + +// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the +// operator's warrant calls the controller's own push with the machine and the modules, once. +func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + f.held = []heldFacts{aHeldMailu()} + c := heldConditionOn(f) + r := newAskerRig(t) + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 1 || len(sent[0].Options) != 1 { + t.Fatalf("asked %+v", sent) + } + o := sent[0].Options[0] + if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" { + t.Errorf("the option is %+v", o) + } + if err := sent[0].Check(r.now); err != nil { + t.Errorf("the ask is refused as made: %v", err) + } + w := r.warrantFor(t, c.Key, "Release") + answerWith(t, r, w) + answerWith(t, r, w) + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") || + !strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) || + !strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) || + !strings.Contains(r.called[0], "chose Release") { + t.Fatalf("called %v", r.called) + } +} + +// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them. +func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) { + for _, verb := range []string{"push", "upgrade"} { + granted := false + for _, v := range broker.VerbsTheControllerActsOnAWarrant { + granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb) + } + if !granted { + t.Errorf("the controller may not act on a warrant for %s", verb) + } + } +} + +func aDoneMailuPlan() inventory.Plan { + return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone, + Tiers: [][]string{{"a"}, {"b"}, {"mailu"}}, + Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc", + Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}} +} + +// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine +// no longer holds it, or holds another build of the module, the line says nothing more. +func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) { + p := aDoneMailuPlan() + line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()}) + if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") { + t.Errorf("the line: %q", line) + } + if said := heldSaid(p, nil); said != "" { + t.Errorf("a plan whose moves were sent says %q", said) + } + other := aHeldMailu() + other.moves[0].To = "9999999ffff" + if said := heldSaid(p, []heldFacts{other}); said != "" { + t.Errorf("a plan older than the build now held says %q", said) + } + p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy} + if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" { + t.Errorf("a move the plan sent says %q", said) + } +} + +// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers +// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant. +func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) { + saved := backlogNow + t.Cleanup(func() { backlogNow = saved }) + backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person" + backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}} + got := backlogObservation() + if len(got) != 1 || len(got[0].Actions) != 1 { + t.Fatalf("raised %+v", got) + } + a := got[0].Actions[0] + if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove || + a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { + t.Fatalf("the action: %+v", a) + } + o := got[0] + c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} + r := newAskerRig(t) + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" { + t.Fatalf("asked %+v", sent) + } + w := r.warrantFor(t, c.Key, "Release") + answerWith(t, r, w) + answerWith(t, r, w) + if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") || + !strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") { + t.Fatalf("called %v", r.called) + } +} + +// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for +// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is. +func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) { + saved := backlogNow + t.Cleanup(func() { backlogNow = saved }) + now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC) + p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding, + Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute), + Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}}, + Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))} + backlogNow.held = "" + if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") { + t.Errorf("the wait is not said: %q", line) + } + backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person" + line := planLine(p, now) + if strings.Contains(line, "LATE") || strings.Contains(line, "building") || + !strings.Contains(line, "until a person releases it") { + t.Errorf("a plan held behind a failed release reads %q", line) + } +} + +// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move +// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it. +func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) + h := aHeldMailu() + h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb", + Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true}) + f.held = []heldFacts{h} + got := watchMovesHeld(f) + if len(got) != 1 || len(got[0].Actions) != 0 { + t.Fatalf("%+v", got) + } + o := got[0] + text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said + for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} { + if !strings.Contains(text, want) { + t.Errorf("does not say %q: %q", want, text) + } + } + w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} + if why, ok := conditions.PlainWords(w, "novox"); !ok { + t.Errorf("not plain: %s", why) + } + f.held = []heldFacts{aHeldMailu()} + if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" { + t.Fatalf("a machine with only asked moves: %+v", got) + } +} diff --git a/cmd/mesh-controller/image_missing.go b/cmd/mesh-controller/image_missing.go new file mode 100644 index 00000000..181314da --- /dev/null +++ b/cmd/mesh-controller/image_missing.go @@ -0,0 +1,245 @@ +package main + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// No declaration is sent that names an image the artifact store does not hold (novox/hq issue 1742). +// +// On 2026-10-21 at 17:13 a send to novox named `mailu/automx@sha256:c62150b8…`; the build record keeps that digest +// ("recent-build") and the store did not hold it. The node-engine removed the running container, then could not pull +// the replacement, and every later apply on novox failed for the next hour while the mail clients' autoconfiguration +// had no backend. The node-engine's half (pull first, replace after) is its own change; this is the controller's: **a +// module whose image the store lacks is left out of the declaration**, as a module whose setting nobody gave is +// (ADR 0163, rule 6) — the node-engine keeps what it has of it and touches none of its containers — and **it is said**, +// row S25: a condition naming the module, the machine and the image, until the store holds the image again (a new build +// of the module, or the copy put back). +// +// Judged only for an image the store's own address names, from the send's own resources; an image of another registry +// is no business of the store's. A store that cannot be asked judges nothing: a send is not refused for a question +// that did not answer. + +// kindImageMissing is S25's kind: module...image-missing. +const kindImageMissing = "image-missing" + +// missingImage is a module held out of a machine's declaration because the store lacks an image it names. +type missingImage struct { + Node, Module, Image string +} + +// storeHolds is whether the store holds the manifest an image reference at its address names. +type storeHolds func(ctx context.Context, reference string) (bool, error) + +// imageInStore is the artifact-store reference of an image the store's address names, "" for any other. +func imageInStore(image, address string) string { + if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") { + return "" + } + reference := catalogue.Recorded(image) + if _, ours := catalogue.InArtifactStore(reference); !ours { + return "" + } + return reference +} + +// holdBackMissingImages leaves out of the declaration every module that names an image the store does not hold, +// and answers what it left out. The module's resources go, it joins LeftOut with the reason, and its build stays the +// one the machine was last sent (`was`). +func holdBackMissingImages(ctx context.Context, node string, s *sendable, address string, holds storeHolds, + was map[string]string) []missingImage { + lacks := map[string]bool{} // reference → the store does not hold it; absent: not asked, or no answer + asked := map[string]bool{} + lacking := map[string]string{} // module → the image + for _, r := range s.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + image, _ := r["image"].(string) + reference := imageInStore(image, address) + module := s.moduleOf[fmt.Sprint(r["id"])] + if reference == "" || module == "" || lacking[module] != "" { + continue + } + if !asked[reference] { + asked[reference] = true + held, err := holds(ctx, reference) + // empty-on-error: the store did not answer, so nothing is held back for a question unanswered + lacks[reference] = err == nil && !held + } + if lacks[reference] { + lacking[module] = image + } + } + if len(lacking) == 0 { + return nil + } + kept := s.Resources[:0:0] + for _, r := range s.Resources { + if _, out := lacking[s.moduleOf[fmt.Sprint(r["id"])]]; !out { + kept = append(kept, r) + } + } + s.Resources = kept + if s.leftOutWhy == nil { + s.leftOutWhy = map[string]string{} + } + var found []missingImage + for module, image := range lacking { + s.leftOutWhy[module] = "its image " + image + " is not in the artifact store" + if !slices.Contains(s.LeftOut, module) { + s.LeftOut = append(s.LeftOut, module) + } + if s.Builds != nil { + if prior, sent := was[module]; sent { + s.Builds[module] = prior + } else { + delete(s.Builds, module) + } + } + found = append(found, missingImage{Node: node, Module: module, Image: image}) + } + sort.Strings(s.LeftOut) + sort.Slice(found, func(i, j int) bool { return found[i].Module < found[j].Module }) + return found +} + +// withoutMissingImages is composeSend's last step: the store asked about the images the declaration names, the modules +// whose image it lacks left out and kept in the ledger S25 reads. A store that cannot be found or asked leaves the +// declaration as composed. +func withoutMissingImages(ctx context.Context, open *stores, node string, s sendable) sendable { + inv := open.inventory + shelf, err := inv.Catalogue(ctx) + if err != nil { + return s + } + address, err := artifactStoreAddress(ctx, inv, shelf, node) + if err != nil || address == "" { + return s + } + sent, known, err := inv.SentBuilds(ctx, node) + if err != nil || !known { + sent = nil + } + store := artifacts.Store{Address: address} + found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent) + for _, m := range found { + fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image) + } + return s +} + +// imageFacts are the images S25 raises: a module whose registered build names an image the store lacks, on each machine +// that runs it. +type imageFacts struct { + missingImage +} + +// imagesHeld remembers the images the store was found to hold: a stored image is named by its digest and stays what it +// is, so it is asked about once; one it lacks is asked again at every look. +var imagesHeld = struct { + sync.Mutex + held map[string]bool +}{held: map[string]bool{}} + +// gatherMissingImages is every machine's module whose registered build — the one a send would carry — names an image of +// the artifact store that the store does not hold. Read from the catalogue and the store, never from what a send did: +// a push is a command of its own, and the serving controller's watchdog hears nothing of it. +func gatherMissingImages(ctx context.Context, inv *inventory.Inventory, holds storeHolds) ([]imageFacts, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, fmt.Errorf("the catalogue cannot be read: %w", err) + } + var out []imageFacts + for _, module := range sortedKeysOf(modulesOf(shelf)) { + for _, r := range shelf[module].Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + image, _ := r["image"].(string) + if _, ours := catalogue.InArtifactStore(image); !ours || !strings.Contains(image, "@sha256:") { + continue + } + imagesHeld.Lock() + known := imagesHeld.held[image] + imagesHeld.Unlock() + if !known { + held, err := holds(ctx, image) + if err != nil { + return nil, fmt.Errorf("the artifact store cannot be asked about %s: %w", image, err) + } + if held { + imagesHeld.Lock() + imagesHeld.held[image] = true + imagesHeld.Unlock() + continue + } + } else { + continue + } + running, err := inv.Running(ctx, module) + if err != nil { + return nil, err + } + for _, node := range running { + out = append(out, imageFacts{missingImage{Node: node, Module: module, Image: image}}) + } + } + } + return out, nil +} + +// modulesOf is the shelf's modules as a map of names, for sortedKeysOf. +func modulesOf(shelf map[string]catalogue.Manifest) map[string]string { + out := make(map[string]string, len(shelf)) + for m := range shelf { + out[m] = m + } + return out +} + +// watchMissingImages is S25: a module left out of a machine's declaration because the store lacks its image, said at +// once. The condition ends when the store holds the image, or a send names another. +func watchMissingImages(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, m := range f.images { + out = append(out, conditions.Observation{Scope: conditions.ScopeModule, ID: m.Module + "." + m.Node, Kind: kindImageMissing, + Machine: m.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+ + "not hold (the evidence names it); the running container is untouched", m.Module, m.Node), + Said: "image " + m.Image + " is not in the store", + Headline: clipWords(conditions.Capital(m.Module)+" is not updated on "+m.Node, conditions.HeadlineMax), + Explanation: clipWords(fmt.Sprintf("The new version of %s names a container image that the mesh's store does not "+ + "hold, so it was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", + m.Module, m.Node, m.Node), conditions.ExplanationMax), + Needs: fmt.Sprintf("build %s again; the details name the image.", m.Module), + Resolved: "Resolved: the store holds the image of " + m.Module + " again"}) + } + return out +} + +// gatherMissingImagesHere is gatherMissingImages asking the mesh's own artifact store; nothing to ask where the mesh has +// none on its network yet. +func gatherMissingImagesHere(ctx context.Context, inv *inventory.Inventory) ([]imageFacts, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, fmt.Errorf("the catalogue cannot be read: %w", err) + } + address, err := artifactStoreAddress(ctx, inv, shelf, "") + if err != nil { + return nil, err + } + if address == "" { + return nil, nil + } + return gatherMissingImages(ctx, inv, artifacts.Store{Address: address}.HoldsManifest) +} diff --git a/cmd/mesh-controller/image_missing_test.go b/cmd/mesh-controller/image_missing_test.go new file mode 100644 index 00000000..86362ea9 --- /dev/null +++ b/cmd/mesh-controller/image_missing_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// **No declaration is sent that names an image the artifact store does not hold** (novox/hq issue 1742). + +const storeAt = "novox.internal:5100" + +// automx is the image the send of 2026-10-21 17:13 named and the store lacked; the others are held. +const ( + automxImage = storeAt + "/mailu/automx@sha256:c62150b8e2cbcc18cb911b098c2877f928761a1e6251667f39ea18251ea35591" + imapImage = storeAt + "/mailu/imap@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +) + +func aMailSend() sendable { + return sendable{ + Resources: []map[string]any{ + {"id": "mailu.automx", "type": "container", "image": automxImage}, + {"id": "mailu.automx-config", "type": "file", "path": "/etc/automx.conf"}, + {"id": "mailu.imap", "type": "container", "image": imapImage}, + {"id": "other.web", "type": "container", "image": "docker.io/library/nginx@sha256:" + strings.Repeat("b", 64)}, + {"id": "dns.dnsmasq", "type": "container", "image": storeAt + "/dnsmasq/main@sha256:" + strings.Repeat("c", 64)}, + }, + moduleOf: map[string]string{"mailu.automx": "automx", "mailu.automx-config": "automx", "mailu.imap": "mailu", + "other.web": "other", "dns.dnsmasq": "dnsmasq"}, + Builds: map[string]string{"automx": "b2f5ebb2new", "mailu": "m2", "other": "o1", "dnsmasq": "d1"}, + } +} + +func storeHolding(missing ...string) (storeHolds, *[]string) { + var asked []string + return func(_ context.Context, reference string) (bool, error) { + asked = append(asked, reference) + for _, m := range missing { + if strings.Contains(reference, m) { + return false, nil + } + } + return true, nil + }, &asked +} + +// 1742/AC1: given a send naming automx's image, which the store does not hold, when the declaration is judged, then +// every resource of automx is out of it, automx is among the modules left out with the reason, its build is the one the +// machine was last sent, and every other module — an image of another registry among them — is as composed. +func TestAModuleWhoseImageTheStoreLacksIsLeftOutOfTheSend(t *testing.T) { + s := aMailSend() + holds, asked := storeHolding("mailu/automx@") + found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, map[string]string{"automx": "b2f5ebb2old"}) + if len(found) != 1 || found[0].Module != "automx" || found[0].Node != "novox" || found[0].Image != automxImage { + t.Fatalf("found %+v", found) + } + var ids []string + for _, r := range s.Resources { + ids = append(ids, r["id"].(string)) + } + if got := strings.Join(ids, ","); got != "mailu.imap,other.web,dns.dnsmasq" { + t.Errorf("the declaration keeps %s", got) + } + if len(s.LeftOut) != 1 || s.LeftOut[0] != "automx" || !strings.Contains(s.leftOutWhy["automx"], "not in the artifact store") { + t.Errorf("left out %v because %v", s.LeftOut, s.leftOutWhy) + } + if s.Builds["automx"] != "b2f5ebb2old" || s.Builds["mailu"] != "m2" || s.Builds["other"] != "o1" { + t.Errorf("builds %v", s.Builds) + } + for _, a := range *asked { + if strings.Contains(a, "docker.io") { + t.Errorf("the store was asked about an image of another registry: %s", a) + } + } +} + +// 1742/AC2: given a module never sent to the machine, when its image is missing, then it has no build in the record of the +// send; given the store does not answer, then nothing is held back for a question unanswered. +func TestAnUnansweredStoreHoldsNothingBackAndANewModuleHasNoBuild(t *testing.T) { + s := aMailSend() + holds, _ := storeHolding("mailu/automx@") + found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, nil) + if len(found) != 1 { + t.Fatalf("%+v", found) + } + if _, has := s.Builds["automx"]; has { + t.Errorf("a module never sent is recorded as sent: %v", s.Builds) + } + s = aMailSend() + silent := func(context.Context, string) (bool, error) { return false, errors.New("registry: connection refused") } + if found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, silent, nil); len(found) != 0 || len(s.LeftOut) != 0 || + len(s.Resources) != 5 { + t.Errorf("an unanswered store held back %+v, left out %v", found, s.LeftOut) + } +} + +// 1742/AC3: a module left out for its image raises a condition for the operator naming the module, the machine and the +// image in the evidence, plain words, and no answer to give; a mesh whose images are all held raises nothing. +func TestAMissingImageIsRaisedForTheOperator(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 40, 0, 0, time.UTC)) + if got := watchMissingImages(f); len(got) != 0 { + t.Fatalf("%+v", got) + } + f.images = []imageFacts{{missingImage{Node: "novox", Module: "automx", Image: automxImage}}} + got := watchMissingImages(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Kind != "image-missing" || o.Key() != "module.automx.novox.image-missing" || o.Resolver != conditions.ResolverOperator || + o.Severity != conditions.Warning || o.Machine != "novox" { + t.Fatalf("%+v (%s)", o, o.Key()) + } + if !strings.Contains(o.Said, "c62150b8") || !strings.Contains(o.Headline+o.Explanation, "automx") { + t.Errorf("words: %q / %q / %q", o.Headline, o.Explanation, o.Said) + } + w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} + if why, ok := conditions.PlainWords(w, "novox"); !ok { + t.Errorf("not plain: %s", why) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 41ebf533..fce97830 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -457,6 +457,18 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "A delivery did not finish. Nothing is tried again until you say so.", Resolved: "Resolved: the delivery is no longer failed"} }), + kindMovesHeld: worded(func(o conditions.Observation) words { + return words{Headline: "An update waits for you", + Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.", + Resolved: "Resolved: the update is no longer waiting"} + }), + kindImageMissing: worded(func(o conditions.Observation) words { + return words{Headline: "A module is not updated", + Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " + + "was not sent. What runs is untouched.", + Needs: "build the module again; the details name the image.", + Resolved: "Resolved: the store holds the image again"} + }), "ask-lost": worded(func(o conditions.Observation) words { return words{Headline: "A build has no result", Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go index 97a9dc5d..653eaf1c 100644 --- a/cmd/mesh-controller/plain_words_test.go +++ b/cmd/mesh-controller/plain_words_test.go @@ -260,7 +260,7 @@ func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) { plainExample(t, got[0], "Updates of openrazer and sensors wait for your release", "Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+ "sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+ - "They are not delivered until then, and stay held if you leave them.") + "They are not delivered until then, and stay held if you leave them.", "Release") } // **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any diff --git a/cmd/mesh-controller/plans_follow.go b/cmd/mesh-controller/plans_follow.go index cf48e194..cae2f5ba 100644 --- a/cmd/mesh-controller/plans_follow.go +++ b/cmd/mesh-controller/plans_follow.go @@ -153,7 +153,7 @@ func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsk line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string { on, _ := inv.Running(ctx, module) return on - }, waitsOf(ctx, inv, one)) + }, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv)) doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line) return json.NewEncoder(os.Stdout).Encode(doc) } diff --git a/cmd/mesh-controller/recorded_moves.go b/cmd/mesh-controller/recorded_moves.go index 156b7466..be6eb18f 100644 --- a/cmd/mesh-controller/recorded_moves.go +++ b/cmd/mesh-controller/recorded_moves.go @@ -181,15 +181,24 @@ func planModuleNames(p inventory.Plan) map[string]string { return out } -// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why -// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a -// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. -func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { +// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6). +type heldMove struct { + Module string + From, To string + // Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold. + Why string + // Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the + // operator is not asked about again: they chose that hold, and the bus has its own condition (S17). + Chosen bool +} + +// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order. +func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) { sent, known, err := inv.SentBuilds(ctx, node) if err != nil || !known { return nil, err } - var out []string + var out []heldMove for _, m := range sortedKeysOf(sent) { was := sent[m] now, held := f.current[m] @@ -197,12 +206,25 @@ func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, continue } if v := f.recordedMove(m, was); !v.rolls { - out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said)) + out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said, + Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus}) } } return out, nil } +// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why +// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a +// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. +func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { + moves, err := heldMovesOn(ctx, inv, f, node) + var out []string + for _, h := range moves { + out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why)) + } + return out, err +} + // stringsKeys is a map of lists as a map of its keys, for sortedKeysOf. func stringsKeys(m map[string][]string) map[string]string { out := map[string]string{} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index ff063b71..348468c8 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -1371,6 +1371,10 @@ func backlogObservation() []conditions.Observation { "them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)} w := releaseHeldWords(modules, machines) o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs + // Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own + // `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged. + o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove, + Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}} return []conditions.Observation{o} } diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 430cb335..85984ec6 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -2061,6 +2061,12 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D // refused ("a build waits there for its gate: (...)") is kept there. "building" with LATE // hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920). what = p.Note + // And when the builds are held after a failed release, they wait for a person, not for a build, and the + // operator is asked to release them (novox/hq issue 1732): said so, and not late. + if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" { + what += " — held after a failed delivery until a person releases it (the operator is asked)" + late = "" + } return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late) } return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late) @@ -2268,7 +2274,7 @@ func plansCommand(ctx context.Context, args []string) error { bounds.of(p.Repository), func(module string) []string { on, _ := inv.Running(ctx, module) return on - }, waitsOf(ctx, inv, []inventory.Plan{p}))) + }, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv))) if r := p.Release; r != nil { // A release plan's walk (ADR 0236): machines done, the one judged, those to come. fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), @@ -2463,11 +2469,12 @@ func plansCommand(ctx context.Context, args []string) error { return on } waits := waitsOf(ctx, inv, plans) + held := heldNow(ctx, inv) for _, p := range plans { if p.Batch() { continue } - fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits)) + fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits), heldSaid(p, held)) } return nil } diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 33a49603..76e65509 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -244,6 +244,22 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since }) }}, + {Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator", + Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)", + Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " + + "machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " + + "pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.", + Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld, + newest: func(f *signalFacts) time.Time { return time.Time{} }}, + {Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names", + Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)", + Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " + + "machine that runs it and the image; every send leaves that module out; the store is asked again at each " + + "look and the condition ends when it holds the image", + Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages, + newest: func(f *signalFacts) time.Time { return time.Time{} }}, {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 488117da..9bf41d09 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -171,6 +171,19 @@ var suppressions = map[string]suppression{ since: f.now.Add(-time.Second)}} }, }, + // A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732). + // Inside: nothing waits there. + "S24": { + inside: func(f *signalFacts) { f.held = nil }, + past: func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} }, + }, + // A registered build naming an image the artifact store lacks is raised at the first look (novox/hq issue 1742). + "S25": { + inside: func(f *signalFacts) { f.images = nil }, + past: func(f *signalFacts) { + f.images = []imageFacts{{missingImage{Node: "anchor", Module: "automx", Image: automxImage}}} + }, + }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index a5f39a8a..c714678d 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -58,6 +58,12 @@ type signalFacts struct { // failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449). failed []failedPlanFacts failedErr error + // held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732). + held []heldFacts + heldErr error + // images are the modules left out of a send because the artifact store lacks an image they name (S25, issue 1742). + images []imageFacts + imagesErr error // refusedSends are the sends a machine refused within the hour for the generation they were composed // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send @@ -357,6 +363,8 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { f.batches, f.plansErr = gatherBatches(ctx, inv, now) } f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) + f.held, f.heldErr = gatherHeldMoves(ctx, inv) + f.images, f.imagesErr = gatherMissingImagesHere(ctx, inv) f.loop, f.loopErr = w.gatherLoop() f.asks, f.asksErr = w.gatherAsks(ctx, inv) if open, err := w.keeper.Open(ctx); err != nil { diff --git a/internal/broker/nats.go b/internal/broker/nats.go index c9007a14..e6ff4265 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -361,9 +361,11 @@ var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"} // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the -// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat. +// controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release +// sent by its own `upgrade release-backlog` (novox/hq issue 1732). Named one by one; a node seat's on any machine, a +// mesh seat's flat. var VerbsTheControllerActsOnAWarrant = []SeatVerb{{Seat: "node-service-manager", Verb: "restart"}, - {Seat: ControllerSeat, Verb: "plans"}} + {Seat: ControllerSeat, Verb: "plans"}, {Seat: ControllerSeat, Verb: "push"}, {Seat: ControllerSeat, Verb: "upgrade"}} // VerbsTheControllerAsksTheOperatorChannel are the operator channel's verbs the controller calls (novox/hq // ADR 0380): `tell`, one plain message with no condition behind it — a module's own secret given, named, @@ -387,6 +389,16 @@ type DegradableGrant struct { // works. Every other grant stays required — missing one of those is a controller that comes up and is // refused on the first thing it tries, with no fallback that covers it. var DegradableGrants = []DegradableGrant{ + { + Subject: "mesh.seat." + ControllerSeat + ".tool.push", + Reason: "refused: the ask of a held move (Release) is answered, and the refusal told, while the operator's own " + + "push at the terminal still sends the machine (issue 1732; ADR 0259)", + }, + { + Subject: "mesh.seat." + ControllerSeat + ".tool.upgrade", + Reason: "refused: the ask of the builds held after a failed release is answered, and the refusal told, while the " + + "operator's own `upgrade release-backlog` at the terminal still releases them (issue 1732; ADR 0259)", + }, { Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb, Reason: "refused: the controller raises the secret-given condition as before (ADR 0380 §9)", diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 1e2b427d..835d59c9 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -29,7 +29,7 @@ accounts { ] users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/builder/registry.go b/internal/builder/registry.go index 4f7ecd11..95918550 100644 --- a/internal/builder/registry.go +++ b/internal/builder/registry.go @@ -59,15 +59,35 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string) }); err != nil { return "", err } - out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{index .RepoDigests 0}}", remote) + // Every digest the daemon lists for the image, one per line: the image may have been pushed under other names before + // (a check's scratch copy, another registry), and the first of them is not the one this push made (novox/hq issue + // 1742: a build recorded a digest the store never held). + out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{range .RepoDigests}}{{println .}}{{end}}", remote) if err != nil { return "", fmt.Errorf("pushed %s and cannot read back what it was pinned as: %w", remote, err) } - pinned := strings.TrimSpace(out) - if !strings.Contains(pinned, "@sha256:") { + first, pinned := "", "" + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if first == "" { + first = line + } + if strings.HasPrefix(line, remote+"@sha256:") { + pinned = line + break + } + } + if pinned == "" { + if strings.Contains(first, "@sha256:") { + return "", fmt.Errorf("pushed %s, but the daemon lists no digest for that name (only %q): the digest of another "+ + "repository is not what this registry serves", remote, first) + } // A tag is not a pin. It can be made to point at something else, and this file is applied // on machines with no mesh to ask about anything (novox/hq ADR 0006). - return "", fmt.Errorf("%s came back as %q, which is not pinned by digest", remote, pinned) + return "", fmt.Errorf("%s came back as %q, which is not pinned by digest", remote, first) } return pinned, nil } diff --git a/internal/builder/registry_test.go b/internal/builder/registry_test.go index 1e3a440d..69ff7c2b 100644 --- a/internal/builder/registry_test.go +++ b/internal/builder/registry_test.go @@ -252,3 +252,32 @@ func TestAPushedImageComesBackPinned(t *testing.T) { t.Fatalf("it did not tag and then push: %v", ran) } } + +// 1742: an image pushed under other names before lists their digests too, and the digest recorded is the one of the name +// this push made — never the first listed, which may be another repository's. +func TestThePinnedDigestIsTheOneOfTheNamePushed(t *testing.T) { + want := "registry.invalid/shell/server@sha256:" + strings.Repeat("b", 64) + earlier := "registry.invalid/scratch/shell/server@sha256:" + strings.Repeat("a", 64) + r := Registry{Address: "registry.invalid", Run: func( + _ context.Context, _, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "inspect" { + return earlier + "\n" + want + "\n", nil + } + return "", nil + }} + got, err := r.PublishImage(context.Background(), "local", "shell/server") + if err != nil || got != want { + t.Fatalf("pinned %q (%v); want %q", got, err, want) + } + // Only another repository's digest is listed: the push is not taken for pinned. + r.Run = func(_ context.Context, _, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "inspect" { + return earlier + "\n", nil + } + return "", nil + } + if _, err := r.PublishImage(context.Background(), "local", "shell/server"); err == nil || + !strings.Contains(err.Error(), "no digest for that name") { + t.Fatalf("a digest of another repository was taken for the push's own: %v", err) + } +}