From e324f5d821627c295b8d5a4922230d24c8594545 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 16:39:47 +0200 Subject: [PATCH] Send an approved settings layer by a walk at once, so the operator's Approve is the act that takes effect (issue 641, ADR 0344) The warrant set the layer and left the push as a second act nobody took: the operator approved a betterbird change on Telegram and nothing reached g14. Now the warrant opens a settings walk: the machines the layer reaches, one at a time, the module judged on each at its gate, its end told once through plan-moved. The ask and the proposer's answer say so instead of naming push. --- cmd/mesh-controller/proposals.go | 57 +++- cmd/mesh-controller/release.go | 11 +- cmd/mesh-controller/settings_walk.go | 234 ++++++++++++++ cmd/mesh-controller/settings_walk_test.go | 359 ++++++++++++++++++++++ internal/inventory/plans.go | 14 + 5 files changed, 659 insertions(+), 16 deletions(-) create mode 100644 cmd/mesh-controller/settings_walk.go create mode 100644 cmd/mesh-controller/settings_walk_test.go diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 6dd4d000..f3c0de6d 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -18,8 +18,9 @@ package main // it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a // person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its // own: nothing consumes one, and the installer's first user list in the node-engine's repository names every -// controller event, so one would cost a node-engine change for a fact without a reader. The push afterwards is a -// separate act, as it is for a layer set at the terminal. +// controller event, so one would cost a node-engine change for a fact without a reader. No push is left for +// afterwards: the warrant that sets the layer opens the walk that sends it (novox/hq ADR 0344, settings_walk.go), +// so the operator's one Approve is the act that makes the change take effect. // // **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed // key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the @@ -47,6 +48,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/outward" ) @@ -179,11 +181,7 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin "this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " + "mesh-cli settings proposals " + id + ".") } - if p.Clear { - b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.") - } else { - b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.") - } + b.WriteString(" " + p.onApprove()) q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator, Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes", About: p.about()} @@ -192,9 +190,9 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin binds, _ := asks.ActDigest(boundAct(act)) oid := optionID(act.Label) options[oid] = i - does := "the settings are set; nothing is pushed yet" + does := "the settings are set and sent to " + p.reaches() + " at once, each machine judged" if p.Clear { - does = "the layer is removed; nothing is pushed yet" + does = "the layer is removed and that is sent to " + p.reaches() + " at once, each machine judged" } if act.Arguments["answer"] == answerDecline { does = "nothing changes; the proposal is discarded" @@ -519,15 +517,28 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error) fingerprint(p.Digest)) fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n", q.Expires.Local().Format("2006-01-02 15:04")) - fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, `push %s` sends it", id, pushWord(p.Node)) + fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, a walk sends it to %s at once, one machine at "+ + "a time, each judged, and the operator is told when it has landed (novox/hq ADR 0344)", id, p.reaches()) return b.String(), nil } -func pushWord(node string) string { - if node == "" { - return "--behind" +// reaches is the machines an approved layer is sent to, in words: the one it names, or every machine running the +// module. +func (p settingsProposal) reaches() string { + if p.Node == "" { + return "every machine running " + p.Module } - return node + return p.Node +} + +// onApprove is what the ask says Approve does (novox/hq ADR 0344 §4): the one Approve sets the layer and sends it. +func (p settingsProposal) onApprove() string { + what := "the layer is set" + if p.Clear { + what = "the layer is removed" + } + return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged; you are told when it "+ + "has landed. A module that reads its settings only when it starts takes them at its next start.", what, p.reaches()) } func newProposalID() string { @@ -788,6 +799,22 @@ func setLayerIn(open *stores) setOnWarrant { changed = "nothing changed" } } - return changed + " (ask " + askID + ")", nil + // **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches + // at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it. + walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID, + Change: changed}) + if err != nil { + return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it", + changed, askID, err, pushTo(p.Node)) + } + return changed + " (ask " + askID + "); " + walked, nil } } + +// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind. +func pushTo(node string) string { + if node == "" { + return "--behind" + } + return node +} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index a97cf4a7..ca9c0bd4 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -616,6 +616,10 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl } for _, p := range plans { if p.Release != nil { + if by != "" && p.Release.Settings != nil { + return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is", + p.ID, p.Repository, p.ID) + } if by != "" { return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID) } @@ -641,7 +645,9 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl return nil, err } for _, p := range recent { - if p.Release == nil { + // A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's + // settings on one machine, said on its own, not a build no gate passed. + if p.Release == nil || p.Release.Settings != nil { continue } if p.State == inventory.PlanFailed { @@ -702,6 +708,9 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl // gate, or the machine being judged judged once more; the plan done when every machine is. func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { r := p.Release + if r.Settings != nil { + return advanceSettingsWalk(ctx, open, p) + } now := time.Now().UTC() if r.Gate == nil { heard, err := releaseHeard(ctx, open) diff --git a/cmd/mesh-controller/settings_walk.go b/cmd/mesh-controller/settings_walk.go new file mode 100644 index 00000000..d55469c5 --- /dev/null +++ b/cmd/mesh-controller/settings_walk.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR +// 0344, issue 641). +// +// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the +// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer +// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the +// builds. Its order is every machine running the module that the layer reaches, the control node last; each is +// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is +// a gated send's, so whatever else waits on that machine for a gate goes with it and is judged with it, and every +// recorded module stays at the build its machine was last sent (ADR 0242): a policy of `record` holds a build, +// never a setting. The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's +// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing +// else. + +// settingsWalkPrefix names a settings walk's id. +const settingsWalkPrefix = "settings-" + +// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository. +func settingsWalkFor(s inventory.PlanSettings) string { + where := s.Node + if where == "" { + where = "the whole mesh" + } + return fmt.Sprintf("the settings of %s on %s", s.Module, where) +} + +// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine +// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure. +func settingsWalkOrder(node string, running, controllers []string) []string { + var order, last []string + for _, n := range running { + if node != "" && n != node { + continue + } + if slices.Contains(controllers, n) { + last = append(last, n) + } else { + order = append(order, n) + } + } + sort.Strings(order) + sort.Strings(last) + return append(order, last...) +} + +// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the +// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant, +// after the layer is set. +var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) { + inv := open.inventory + running, err := inv.Running(ctx, s.Module) + if err != nil { + return "", err + } + controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) + if err != nil { + return "", err + } + order := settingsWalkOrder(s.Node, running, controllers) + if len(order) == 0 { + return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil + } + release, err := inv.HoldPlans(ctx, true) + if err != nil { + return "", err + } + defer release() + now := time.Now().UTC() + p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s), + Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{}, + Release: &inventory.PlanRelease{Order: order, Settings: &s}, + Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask, + strings.Join(order, ", "))} + if err := inv.SavePlan(ctx, &p); err != nil { + return "", err + } + fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note) + return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil +} + +// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A +// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration +// that walk's gate reads the machine's report against. +var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) { + plans, err := open.inventory.OpenPlans(ctx) + if err != nil { + return "", err + } + return judgingIn(plans, self, node), nil +} + +// judgingIn is judgingOn over plans read. Pure. +func judgingIn(plans []inventory.Plan, self, node string) string { + judging := func(g *inventory.PlanGate) bool { + return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node) + } + for _, p := range plans { + if p.ID == self || !p.Open() { + continue + } + if p.Release != nil { + if judging(p.Release.Gate) { + return p.ID + } + continue + } + for _, s := range p.Modules { + if s != nil && judging(s.Gate) { + return p.ID + } + } + } + return "" +} + +// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the +// machine being judged judged once more; done when every machine is. +func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { + r, s := p.Release, p.Release.Settings + now := time.Now().UTC() + wait := func(note string) (bool, error) { + changed := p.Note != note + p.Note = note + return changed, nil + } + if r.Gate == nil { + heard, err := releaseHeard(ctx, open) + if err != nil { + return false, err + } + for r.Next < len(r.Order) { + node := r.Order[r.Next] + if !heard[node] { + r.Skipped = append(r.Skipped, node) + r.Next++ + continue + } + if id, err := judgingOn(ctx, open, p.ID, node); err != nil { + return false, err + } else if id != "" { + return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id)) + } + moves, sent, grants, err := gatedSend(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + return wait(fmt.Sprintf("waiting before %s: %v", node, err)) + } + if err == nil && len(sent) == 0 { + // Nothing waited there for a gate: the machine is sent for the layer alone, judged all the same. + sent, grants, err = sendJudged(ctx, open, node) + } + if err != nil { + return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err) + } + r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, Carried: moves, + Sent: sentNow(ctx, open.inventory, sent), Grants: grants} + p.Note = fmt.Sprintf("sent %s %s (ask %s); judging %s there", node, settingsWalkFor(*s), s.Ask, s.Module) + if len(moves) > 0 { + p.Note += fmt.Sprintf(", and %d build(s) that waited there for a gate", len(moves)) + } + if said := grantsSaid(grants); said != "" { + p.Note += "; " + said + } + if said := recreationsSaid(moves); said != "" { + p.Note += "; " + said + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + p.State, p.Tier = inventory.PlanDone, len(p.Tiers) + p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, orNone(strings.Join(r.Done, ", "))) + if len(r.Skipped) > 0 { + p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ") + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + g := r.Gate + node := r.Order[r.Next] + verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now) + if err != nil { + return false, err + } + switch verdict { + case "": + // What the send carried and broke is put back at once, as a judged send's is; the module whose settings + // changed has no build to put back and is judged to its verdict. + if returned := len(g.Returned); putBackBroken(ctx, open, p, g, "", nil) && len(g.Returned) > returned { + return true, nil + } + return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g))) + case inventory.GatePassed: + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, node) + r.Next++ + r.Gate = nil + return true, nil + case inventory.GateSuperseded: + p.State = inventory.PlanSuperseded + p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+ + "machines not yet sent take it at their next send", node, g.Why) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + p.Note = "" + batched, back := batchingRollbacks(ctx) + failCarried(batched, open, p, g, "") + sendRollbacks(ctx, open, p, back) + putBack := p.Note + p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+ + "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + if rest := r.Order[r.Next+1:]; len(rest) > 0 { + p.Note += "; not sent: " + strings.Join(rest, ", ") + } + if putBack != "" { + p.Note += "; " + putBack + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil +} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go new file mode 100644 index 00000000..8cf8e580 --- /dev/null +++ b/cmd/mesh-controller/settings_walk_test.go @@ -0,0 +1,359 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take +// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a +// time, judging the module on each; the ask says so before the Approve, and nothing names a push. + +// The order: every machine running the module, or the one the layer names, the control node last. +func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) { + running := []string{"shanks", "novox", "g14", "ace"} + for _, c := range []struct { + node string + want []string + }{ + {"", []string{"ace", "g14", "shanks", "novox"}}, + {"g14", []string{"g14"}}, + {"novox", []string{"novox"}}, + {"elsewhere", nil}, + } { + if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) { + t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want) + } + } +} + +// A settings walk sends a machine only when no other walk judges there. +func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) { + judging := &inventory.PlanGate{Machines: []string{"g14"}} + passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed} + plans := []inventory.Plan{ + {ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}}, + {ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}}, + {ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, + {ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, + } + if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" { + t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id) + } + if id := judgingIn(plans[1:2], "x", "g14"); id != "" { + t.Errorf("a passed gate judges: %q", id) + } + if id := judgingIn(plans, "x", "shanks"); id != "" { + t.Errorf("on shanks %q judges", id) + } +} + +// The ask says, before the Approve, that the layer is sent at once and judged, and that the operator is told when +// it has landed; neither the ask nor the proposer's answer names a push any more. +func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { + for _, c := range []struct { + in proposeInput + where string + had bool + }{ + {proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false}, + {proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true}, + {proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true}, + } { + r := newProposerRig(t) + if c.had { + r.before, r.had = map[string]any{"x": "0"}, true + } + words, err := r.pr.propose(context.Background(), c.in) + if err != nil { + t.Fatal(err) + } + q := r.askSent(t) + if err := q.Check(r.now); err != nil { + t.Fatalf("the ask is refused: %v", err) + } + if !strings.Contains(q.Explanation, "and sent at once to "+c.where+", one machine at a time, each judged; you "+ + "are told when it has landed") { + t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation) + } + approve := q.Options[0] + if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") { + t.Errorf("Approve does %q", approve.Does) + } + if !strings.Contains(words, "a walk sends it to "+c.where+" at once") { + t.Errorf("the proposer's answer:\n%s", words) + } + for _, said := range []string{q.Explanation, approve.Does, words} { + if strings.Contains(said, "push") { + t.Errorf("a push is still named, as if a second act were needed:\n%s", said) + } + } + } +} + +// ---- on the real stores ----------------------------------------------------------------------------- + +// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a +// machine that does not run the module opens none and says so. +func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "x = ${setting:x}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + set := setLayerIn(open) + now := time.Now() + values := map[string]any{"x": "1"} + p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { + t.Errorf("the warrant's words: %q", words) + } + plans, err := open.inventory.OpenPlans(ctx) + if err != nil || len(plans) != 1 { + t.Fatalf("open walks: %+v %v", plans, err) + } + w := plans[0] + if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || + !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || + !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { + t.Fatalf("the settings walk: %+v %+v", w, w.Release) + } + + // A machine's layer where the module does not run: set, and nothing to send. + q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") + if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { + t.Fatalf("a layer nobody runs: %q %v", words, err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { + t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) + } + + // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. + was := openSettingsWalk + openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { + return "", fmt.Errorf("the plans are not to be had") + } + t.Cleanup(func() { openSettingsWalk = was }) + again := map[string]any{"x": "2"} + r := settingsProposal{Module: "notes", Node: "laptop", Values: again, From: "g14/claude-code", At: now, + Digest: layerDigest(again), BeforeDigest: layerDigest(nil)} + if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ + "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { + t.Errorf("an unopened walk: %v", err) + } +} + +// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at +// once, recorded, and judged from the reports, with a machine named broken failing what it was sent. +type walkMesh struct { + open *stores + sent [][]string + broken map[string]bool +} + +func aSettledMesh(t *testing.T) *walkMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := &walkMesh{open: open, broken: map[string]bool{}} + withConditionsInMemory(t) + was := doctorFrom + doctorFrom = nil + t.Cleanup(func() { doctorFrom = was }) + old := time.Now().Add(-2 * time.Hour) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true}, nil + } + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + w.sent = append(w.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + modules, _ := open.inventory.Assigned(ctx, node) + carried := map[string]string{} + for _, m := range modules { + carried[m] = current[m].Commit + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if w.broken[r.Node] { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + return f, nil + } + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + gateSettle, gateEvery = 0, time.Hour + t.Cleanup(func() { + machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + }) + return w +} + +func (w *walkMesh) walk(t *testing.T) inventory.Plan { + t.Helper() + plans, err := w.open.inventory.RecentPlans(t.Context(), 10) + if err != nil { + t.Fatal(err) + } + for _, p := range plans { + if p.Release != nil && p.Release.Settings != nil { + return p + } + } + t.Fatal("no settings walk") + return inventory.Plan{} +} + +// The walk sends the first machine alone and judges the module there before it sends the next, and ends +// delivered on both; nothing waited for a gate, so each send carried the layer alone. +func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) + if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { + t.Fatalf("%q %v", words, err) + } + advancePlans(ctx, w.open) + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) + } + p := w.walk(t) + if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { + t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) + } + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", w.sent) + } + p = w.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || + p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { + t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) + } + // It said its start and its end, so the operator is told once (ADR 0308). + if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { + t.Fatalf("the walk's start and end are not marked: %+v", p.Times) + } + // Nothing waits, and no walk of waiting builds opened beside it. + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { + t.Fatalf("%+v %v", r, err) + } +} + +// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the +// failure holds back no walk of waiting builds. +func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + w.broken["anchor"] = true + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || + !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) + } + // A build that waits for a gate is still walked: the failed settings walk holds nothing back. + w.broken["anchor"] = false + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index f2293280..c17344a0 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -433,6 +433,20 @@ type PlanRelease struct { Skipped []string `json:"skipped,omitempty"` // By is the person who released it, empty when the mesh did. By string `json:"by,omitempty"` + // Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings + // layer the operator approved, sent to the machines it reaches one at a time, its module judged on each. + Settings *PlanSettings `json:"settings,omitempty"` +} + +// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the +// layer's scope, the ask that approved it and what changed, in words. +type PlanSettings struct { + Module string `json:"module"` + // Node is the machine the layer is for, empty for the whole mesh. + Node string `json:"node,omitempty"` + // Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs"). + Ask string `json:"ask"` + Change string `json:"change,omitempty"` } // The states a plan passes through.