From e324f5d821627c295b8d5a4922230d24c8594545 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 16:39:47 +0200 Subject: [PATCH 1/4] Send an approved settings layer by a walk at once, so the operator's Approve is the act that takes effect (issue 641, ADR 0344) The warrant set the layer and left the push as a second act nobody took: the operator approved a betterbird change on Telegram and nothing reached g14. Now the warrant opens a settings walk: the machines the layer reaches, one at a time, the module judged on each at its gate, its end told once through plan-moved. The ask and the proposer's answer say so instead of naming push. --- cmd/mesh-controller/proposals.go | 57 +++- cmd/mesh-controller/release.go | 11 +- cmd/mesh-controller/settings_walk.go | 234 ++++++++++++++ cmd/mesh-controller/settings_walk_test.go | 359 ++++++++++++++++++++++ internal/inventory/plans.go | 14 + 5 files changed, 659 insertions(+), 16 deletions(-) create mode 100644 cmd/mesh-controller/settings_walk.go create mode 100644 cmd/mesh-controller/settings_walk_test.go diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 6dd4d000..f3c0de6d 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -18,8 +18,9 @@ package main // it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a // person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its // own: nothing consumes one, and the installer's first user list in the node-engine's repository names every -// controller event, so one would cost a node-engine change for a fact without a reader. The push afterwards is a -// separate act, as it is for a layer set at the terminal. +// controller event, so one would cost a node-engine change for a fact without a reader. No push is left for +// afterwards: the warrant that sets the layer opens the walk that sends it (novox/hq ADR 0344, settings_walk.go), +// so the operator's one Approve is the act that makes the change take effect. // // **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed // key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the @@ -47,6 +48,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/outward" ) @@ -179,11 +181,7 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin "this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " + "mesh-cli settings proposals " + id + ".") } - if p.Clear { - b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.") - } else { - b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.") - } + b.WriteString(" " + p.onApprove()) q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator, Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes", About: p.about()} @@ -192,9 +190,9 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin binds, _ := asks.ActDigest(boundAct(act)) oid := optionID(act.Label) options[oid] = i - does := "the settings are set; nothing is pushed yet" + does := "the settings are set and sent to " + p.reaches() + " at once, each machine judged" if p.Clear { - does = "the layer is removed; nothing is pushed yet" + does = "the layer is removed and that is sent to " + p.reaches() + " at once, each machine judged" } if act.Arguments["answer"] == answerDecline { does = "nothing changes; the proposal is discarded" @@ -519,15 +517,28 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error) fingerprint(p.Digest)) fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n", q.Expires.Local().Format("2006-01-02 15:04")) - fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, `push %s` sends it", id, pushWord(p.Node)) + fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, a walk sends it to %s at once, one machine at "+ + "a time, each judged, and the operator is told when it has landed (novox/hq ADR 0344)", id, p.reaches()) return b.String(), nil } -func pushWord(node string) string { - if node == "" { - return "--behind" +// reaches is the machines an approved layer is sent to, in words: the one it names, or every machine running the +// module. +func (p settingsProposal) reaches() string { + if p.Node == "" { + return "every machine running " + p.Module } - return node + return p.Node +} + +// onApprove is what the ask says Approve does (novox/hq ADR 0344 §4): the one Approve sets the layer and sends it. +func (p settingsProposal) onApprove() string { + what := "the layer is set" + if p.Clear { + what = "the layer is removed" + } + return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged; you are told when it "+ + "has landed. A module that reads its settings only when it starts takes them at its next start.", what, p.reaches()) } func newProposalID() string { @@ -788,6 +799,22 @@ func setLayerIn(open *stores) setOnWarrant { changed = "nothing changed" } } - return changed + " (ask " + askID + ")", nil + // **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches + // at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it. + walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID, + Change: changed}) + if err != nil { + return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it", + changed, askID, err, pushTo(p.Node)) + } + return changed + " (ask " + askID + "); " + walked, nil } } + +// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind. +func pushTo(node string) string { + if node == "" { + return "--behind" + } + return node +} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index a97cf4a7..ca9c0bd4 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -616,6 +616,10 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl } for _, p := range plans { if p.Release != nil { + if by != "" && p.Release.Settings != nil { + return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is", + p.ID, p.Repository, p.ID) + } if by != "" { return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID) } @@ -641,7 +645,9 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl return nil, err } for _, p := range recent { - if p.Release == nil { + // A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's + // settings on one machine, said on its own, not a build no gate passed. + if p.Release == nil || p.Release.Settings != nil { continue } if p.State == inventory.PlanFailed { @@ -702,6 +708,9 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl // gate, or the machine being judged judged once more; the plan done when every machine is. func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { r := p.Release + if r.Settings != nil { + return advanceSettingsWalk(ctx, open, p) + } now := time.Now().UTC() if r.Gate == nil { heard, err := releaseHeard(ctx, open) diff --git a/cmd/mesh-controller/settings_walk.go b/cmd/mesh-controller/settings_walk.go new file mode 100644 index 00000000..d55469c5 --- /dev/null +++ b/cmd/mesh-controller/settings_walk.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR +// 0344, issue 641). +// +// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the +// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer +// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the +// builds. Its order is every machine running the module that the layer reaches, the control node last; each is +// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is +// a gated send's, so whatever else waits on that machine for a gate goes with it and is judged with it, and every +// recorded module stays at the build its machine was last sent (ADR 0242): a policy of `record` holds a build, +// never a setting. The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's +// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing +// else. + +// settingsWalkPrefix names a settings walk's id. +const settingsWalkPrefix = "settings-" + +// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository. +func settingsWalkFor(s inventory.PlanSettings) string { + where := s.Node + if where == "" { + where = "the whole mesh" + } + return fmt.Sprintf("the settings of %s on %s", s.Module, where) +} + +// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine +// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure. +func settingsWalkOrder(node string, running, controllers []string) []string { + var order, last []string + for _, n := range running { + if node != "" && n != node { + continue + } + if slices.Contains(controllers, n) { + last = append(last, n) + } else { + order = append(order, n) + } + } + sort.Strings(order) + sort.Strings(last) + return append(order, last...) +} + +// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the +// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant, +// after the layer is set. +var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) { + inv := open.inventory + running, err := inv.Running(ctx, s.Module) + if err != nil { + return "", err + } + controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) + if err != nil { + return "", err + } + order := settingsWalkOrder(s.Node, running, controllers) + if len(order) == 0 { + return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil + } + release, err := inv.HoldPlans(ctx, true) + if err != nil { + return "", err + } + defer release() + now := time.Now().UTC() + p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s), + Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{}, + Release: &inventory.PlanRelease{Order: order, Settings: &s}, + Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask, + strings.Join(order, ", "))} + if err := inv.SavePlan(ctx, &p); err != nil { + return "", err + } + fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note) + return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil +} + +// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A +// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration +// that walk's gate reads the machine's report against. +var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) { + plans, err := open.inventory.OpenPlans(ctx) + if err != nil { + return "", err + } + return judgingIn(plans, self, node), nil +} + +// judgingIn is judgingOn over plans read. Pure. +func judgingIn(plans []inventory.Plan, self, node string) string { + judging := func(g *inventory.PlanGate) bool { + return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node) + } + for _, p := range plans { + if p.ID == self || !p.Open() { + continue + } + if p.Release != nil { + if judging(p.Release.Gate) { + return p.ID + } + continue + } + for _, s := range p.Modules { + if s != nil && judging(s.Gate) { + return p.ID + } + } + } + return "" +} + +// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the +// machine being judged judged once more; done when every machine is. +func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { + r, s := p.Release, p.Release.Settings + now := time.Now().UTC() + wait := func(note string) (bool, error) { + changed := p.Note != note + p.Note = note + return changed, nil + } + if r.Gate == nil { + heard, err := releaseHeard(ctx, open) + if err != nil { + return false, err + } + for r.Next < len(r.Order) { + node := r.Order[r.Next] + if !heard[node] { + r.Skipped = append(r.Skipped, node) + r.Next++ + continue + } + if id, err := judgingOn(ctx, open, p.ID, node); err != nil { + return false, err + } else if id != "" { + return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id)) + } + moves, sent, grants, err := gatedSend(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + return wait(fmt.Sprintf("waiting before %s: %v", node, err)) + } + if err == nil && len(sent) == 0 { + // Nothing waited there for a gate: the machine is sent for the layer alone, judged all the same. + sent, grants, err = sendJudged(ctx, open, node) + } + if err != nil { + return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err) + } + r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, Carried: moves, + Sent: sentNow(ctx, open.inventory, sent), Grants: grants} + p.Note = fmt.Sprintf("sent %s %s (ask %s); judging %s there", node, settingsWalkFor(*s), s.Ask, s.Module) + if len(moves) > 0 { + p.Note += fmt.Sprintf(", and %d build(s) that waited there for a gate", len(moves)) + } + if said := grantsSaid(grants); said != "" { + p.Note += "; " + said + } + if said := recreationsSaid(moves); said != "" { + p.Note += "; " + said + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + p.State, p.Tier = inventory.PlanDone, len(p.Tiers) + p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, orNone(strings.Join(r.Done, ", "))) + if len(r.Skipped) > 0 { + p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ") + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + g := r.Gate + node := r.Order[r.Next] + verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now) + if err != nil { + return false, err + } + switch verdict { + case "": + // What the send carried and broke is put back at once, as a judged send's is; the module whose settings + // changed has no build to put back and is judged to its verdict. + if returned := len(g.Returned); putBackBroken(ctx, open, p, g, "", nil) && len(g.Returned) > returned { + return true, nil + } + return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g))) + case inventory.GatePassed: + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, node) + r.Next++ + r.Gate = nil + return true, nil + case inventory.GateSuperseded: + p.State = inventory.PlanSuperseded + p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+ + "machines not yet sent take it at their next send", node, g.Why) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + p.Note = "" + batched, back := batchingRollbacks(ctx) + failCarried(batched, open, p, g, "") + sendRollbacks(ctx, open, p, back) + putBack := p.Note + p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+ + "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + if rest := r.Order[r.Next+1:]; len(rest) > 0 { + p.Note += "; not sent: " + strings.Join(rest, ", ") + } + if putBack != "" { + p.Note += "; " + putBack + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil +} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go new file mode 100644 index 00000000..8cf8e580 --- /dev/null +++ b/cmd/mesh-controller/settings_walk_test.go @@ -0,0 +1,359 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take +// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a +// time, judging the module on each; the ask says so before the Approve, and nothing names a push. + +// The order: every machine running the module, or the one the layer names, the control node last. +func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) { + running := []string{"shanks", "novox", "g14", "ace"} + for _, c := range []struct { + node string + want []string + }{ + {"", []string{"ace", "g14", "shanks", "novox"}}, + {"g14", []string{"g14"}}, + {"novox", []string{"novox"}}, + {"elsewhere", nil}, + } { + if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) { + t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want) + } + } +} + +// A settings walk sends a machine only when no other walk judges there. +func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) { + judging := &inventory.PlanGate{Machines: []string{"g14"}} + passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed} + plans := []inventory.Plan{ + {ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}}, + {ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}}, + {ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, + {ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, + } + if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" { + t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id) + } + if id := judgingIn(plans[1:2], "x", "g14"); id != "" { + t.Errorf("a passed gate judges: %q", id) + } + if id := judgingIn(plans, "x", "shanks"); id != "" { + t.Errorf("on shanks %q judges", id) + } +} + +// The ask says, before the Approve, that the layer is sent at once and judged, and that the operator is told when +// it has landed; neither the ask nor the proposer's answer names a push any more. +func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { + for _, c := range []struct { + in proposeInput + where string + had bool + }{ + {proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false}, + {proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true}, + {proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true}, + } { + r := newProposerRig(t) + if c.had { + r.before, r.had = map[string]any{"x": "0"}, true + } + words, err := r.pr.propose(context.Background(), c.in) + if err != nil { + t.Fatal(err) + } + q := r.askSent(t) + if err := q.Check(r.now); err != nil { + t.Fatalf("the ask is refused: %v", err) + } + if !strings.Contains(q.Explanation, "and sent at once to "+c.where+", one machine at a time, each judged; you "+ + "are told when it has landed") { + t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation) + } + approve := q.Options[0] + if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") { + t.Errorf("Approve does %q", approve.Does) + } + if !strings.Contains(words, "a walk sends it to "+c.where+" at once") { + t.Errorf("the proposer's answer:\n%s", words) + } + for _, said := range []string{q.Explanation, approve.Does, words} { + if strings.Contains(said, "push") { + t.Errorf("a push is still named, as if a second act were needed:\n%s", said) + } + } + } +} + +// ---- on the real stores ----------------------------------------------------------------------------- + +// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a +// machine that does not run the module opens none and says so. +func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "x = ${setting:x}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + set := setLayerIn(open) + now := time.Now() + values := map[string]any{"x": "1"} + p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { + t.Errorf("the warrant's words: %q", words) + } + plans, err := open.inventory.OpenPlans(ctx) + if err != nil || len(plans) != 1 { + t.Fatalf("open walks: %+v %v", plans, err) + } + w := plans[0] + if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || + !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || + !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { + t.Fatalf("the settings walk: %+v %+v", w, w.Release) + } + + // A machine's layer where the module does not run: set, and nothing to send. + q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") + if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { + t.Fatalf("a layer nobody runs: %q %v", words, err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { + t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) + } + + // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. + was := openSettingsWalk + openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { + return "", fmt.Errorf("the plans are not to be had") + } + t.Cleanup(func() { openSettingsWalk = was }) + again := map[string]any{"x": "2"} + r := settingsProposal{Module: "notes", Node: "laptop", Values: again, From: "g14/claude-code", At: now, + Digest: layerDigest(again), BeforeDigest: layerDigest(nil)} + if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ + "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { + t.Errorf("an unopened walk: %v", err) + } +} + +// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at +// once, recorded, and judged from the reports, with a machine named broken failing what it was sent. +type walkMesh struct { + open *stores + sent [][]string + broken map[string]bool +} + +func aSettledMesh(t *testing.T) *walkMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := &walkMesh{open: open, broken: map[string]bool{}} + withConditionsInMemory(t) + was := doctorFrom + doctorFrom = nil + t.Cleanup(func() { doctorFrom = was }) + old := time.Now().Add(-2 * time.Hour) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true}, nil + } + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + w.sent = append(w.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + modules, _ := open.inventory.Assigned(ctx, node) + carried := map[string]string{} + for _, m := range modules { + carried[m] = current[m].Commit + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if w.broken[r.Node] { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + return f, nil + } + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + gateSettle, gateEvery = 0, time.Hour + t.Cleanup(func() { + machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + }) + return w +} + +func (w *walkMesh) walk(t *testing.T) inventory.Plan { + t.Helper() + plans, err := w.open.inventory.RecentPlans(t.Context(), 10) + if err != nil { + t.Fatal(err) + } + for _, p := range plans { + if p.Release != nil && p.Release.Settings != nil { + return p + } + } + t.Fatal("no settings walk") + return inventory.Plan{} +} + +// The walk sends the first machine alone and judges the module there before it sends the next, and ends +// delivered on both; nothing waited for a gate, so each send carried the layer alone. +func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) + if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { + t.Fatalf("%q %v", words, err) + } + advancePlans(ctx, w.open) + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) + } + p := w.walk(t) + if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { + t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) + } + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", w.sent) + } + p = w.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || + p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { + t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) + } + // It said its start and its end, so the operator is told once (ADR 0308). + if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { + t.Fatalf("the walk's start and end are not marked: %+v", p.Times) + } + // Nothing waits, and no walk of waiting builds opened beside it. + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { + t.Fatalf("%+v %v", r, err) + } +} + +// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the +// failure holds back no walk of waiting builds. +func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + w.broken["anchor"] = true + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || + !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) + } + // A build that waits for a gate is still walked: the failed settings walk holds nothing back. + w.broken["anchor"] = false + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index f2293280..c17344a0 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -433,6 +433,20 @@ type PlanRelease struct { Skipped []string `json:"skipped,omitempty"` // By is the person who released it, empty when the mesh did. By string `json:"by,omitempty"` + // Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings + // layer the operator approved, sent to the machines it reaches one at a time, its module judged on each. + Settings *PlanSettings `json:"settings,omitempty"` +} + +// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the +// layer's scope, the ask that approved it and what changed, in words. +type PlanSettings struct { + Module string `json:"module"` + // Node is the machine the layer is for, empty for the whole mesh. + Node string `json:"node,omitempty"` + // Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs"). + Ask string `json:"ask"` + Change string `json:"change,omitempty"` } // The states a plan passes through. From 1219ec822943e1f15b1f89a4b9ac15831e7560db Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 16:50:10 +0200 Subject: [PATCH 2/4] A settings walk sends the layer alone, every build kept, and stops before a data move (issue 641, ADR 0344) A judged send carries what waits on its machine, which would have released builds held after a failed walk without a person's word. The settings walk now keeps every build of its machine, refuses a send that would move a running module's data, and never says delivered when it reached no machine. --- cmd/mesh-controller/proposals.go | 7 +- cmd/mesh-controller/recorded_kept.go | 26 ++++- cmd/mesh-controller/settings_walk.go | 86 +++++++++------ cmd/mesh-controller/settings_walk_test.go | 127 ++++++++++++++++++++-- 4 files changed, 197 insertions(+), 49 deletions(-) diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index f3c0de6d..fc5db024 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -518,7 +518,7 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error) fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n", q.Expires.Local().Format("2006-01-02 15:04")) fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, a walk sends it to %s at once, one machine at "+ - "a time, each judged, and the operator is told when it has landed (novox/hq ADR 0344)", id, p.reaches()) + "a time, each judged, and its end is told as every walk's is (novox/hq ADR 0344)", id, p.reaches()) return b.String(), nil } @@ -537,8 +537,9 @@ func (p settingsProposal) onApprove() string { if p.Clear { what = "the layer is removed" } - return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged; you are told when it "+ - "has landed. A module that reads its settings only when it starts takes them at its next start.", what, p.reaches()) + return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged, every build there "+ + "kept; its end is told as every walk's is. A module that reads its settings only when it starts takes them at "+ + "its next start.", what, p.reaches()) } func newProposalID() string { diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go index ecf9e25c..2b65aa4b 100644 --- a/cmd/mesh-controller/recorded_kept.go +++ b/cmd/mesh-controller/recorded_kept.go @@ -83,6 +83,22 @@ func everyBuildKept(ctx context.Context, node string) bool { return on != "" && on == node } +type keepForSettingsKey struct{} + +// keepingBuildsForSettings is a context whose sends compose every module of one machine — recorded or rolling out — +// at the build that machine was last sent, as keepingEveryBuild does, for a settings walk's send (novox/hq ADR 0344): +// the layer the operator approved, and nothing of any module's new code. A build held for a person, or waiting +// for a gate, stays where it is; the walk that is its own carries it. +func keepingBuildsForSettings(ctx context.Context, node string) context.Context { + return context.WithValue(ctx, keepForSettingsKey{}, node) +} + +// settingsKeptOn is whether this context is a settings walk's send to this machine. +func settingsKeptOn(ctx context.Context, node string) bool { + on, _ := ctx.Value(keepForSettingsKey{}).(string) + return on != "" && on == node +} + // recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a // build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context // keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere — @@ -92,7 +108,7 @@ func everyBuildKept(ctx context.Context, node string) bool { // 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and // composing the mesh's builds would be the very move the grants step must not make. func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) { - every := everyBuildKept(ctx, node) + every := everyBuildKept(ctx, node) || settingsKeptOn(ctx, node) skip, on := keptExcept(ctx) if !on && !every { return nil, nil @@ -106,6 +122,10 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st return nil, err } if !known { + if settingsKeptOn(ctx, node) { + return nil, fmt.Errorf("what %s was last sent is not known, so its settings cannot be sent there with "+ + "every build kept (novox/hq ADR 0344)", node) + } if every { return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+ "there alone with every build kept (novox/hq issue 490)", node) @@ -166,6 +186,10 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri if err != nil { return nil, err } + if !found && settingsKeptOn(ctx, node) { + return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: its settings cannot "+ + "be sent there with it kept (novox/hq ADR 0344)", node, m, short(kept[m])) + } if !found && everyBuildKept(ctx, node) { return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ "list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m])) diff --git a/cmd/mesh-controller/settings_walk.go b/cmd/mesh-controller/settings_walk.go index d55469c5..cd756b3e 100644 --- a/cmd/mesh-controller/settings_walk.go +++ b/cmd/mesh-controller/settings_walk.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "slices" "sort" @@ -21,9 +20,9 @@ import ( // opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the // builds. Its order is every machine running the module that the layer reaches, the control node last; each is // sent and its module judged there, from the machine's report on that send, before the next is sent. The send is -// a gated send's, so whatever else waits on that machine for a gate goes with it and is judged with it, and every -// recorded module stays at the build its machine was last sent (ADR 0242): a policy of `record` holds a build, -// never a setting. The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's +// the layer alone: every module there stays at the build its machine was last sent (keepingBuildsForSettings), so a +// build held for a person (ADR 0242) or waiting for a gate stays where it is — a policy of `record` holds a build, +// never a setting — and a send that would move a running module's data stops the walk (ADR 0217 §3). The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's // channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing // else. @@ -93,6 +92,29 @@ var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanS return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil } +// settingsDataMoves is what a settings walk's send to a machine would move of a running module's data: composed as +// that send composes it, every build kept, and compared with what the machine was last sent (ADR 0217 §3). +var settingsDataMoves = func(ctx context.Context, open *stores, node string) ([]dataMove, error) { + keeping := keepingBuildsForSettings(keepingRecorded(ctx), node) + gens, err := generators(keeping, open) + if err != nil { + return nil, err + } + plan, settings, err := planFor(keeping, open, node) + if err != nil { + return nil, err + } + declared, err := declarationWith(keeping, open, node, plan, settings, gens, Reading) + if err != nil { + return nil, err + } + body, err := declared.Body() + if err != nil { + return nil, err + } + return heldMoves(keeping, open.inventory, node, body, nil) +} + // judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A // settings walk sends a machine only when no other walk judges there: its send would set aside the declaration // that walk's gate reads the machine's report against. @@ -155,34 +177,43 @@ func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) ( } else if id != "" { return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id)) } - moves, sent, grants, err := gatedSend(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - return wait(fmt.Sprintf("waiting before %s: %v", node, err)) - } - if err == nil && len(sent) == 0 { - // Nothing waited there for a gate: the machine is sent for the layer alone, judged all the same. - sent, grants, err = sendJudged(ctx, open, node) + // **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before + // that machine, failed and said, and `push --move` sends it. + if held, err := settingsDataMoves(ctx, open, node); err != nil { + return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err) + } else if len(held) > 0 { + var said strings.Builder + sayDataHeld(&said, node, held) + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s not sent to %s: a running module's data would move there — %s; the layer "+ + "stays as the operator approved it", settingsWalkFor(*s), node, strings.Join(strings.Fields(said.String()), " ")) + if rest := r.Order[r.Next+1:]; len(rest) > 0 { + p.Note += "; not sent: " + strings.Join(rest, ", ") + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil } + sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node) if err != nil { return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err) } - r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, Carried: moves, + r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, Sent: sentNow(ctx, open.inventory, sent), Grants: grants} - p.Note = fmt.Sprintf("sent %s %s (ask %s); judging %s there", node, settingsWalkFor(*s), s.Ask, s.Module) - if len(moves) > 0 { - p.Note += fmt.Sprintf(", and %d build(s) that waited there for a gate", len(moves)) - } + p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node, + settingsWalkFor(*s), s.Ask, s.Module) if said := grantsSaid(grants); said != "" { p.Note += "; " + said } - if said := recreationsSaid(moves); said != "" { - p.Note += "; " + said - } fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil } p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, orNone(strings.Join(r.Done, ", "))) + p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, strings.Join(r.Done, ", ")) + if len(r.Done) == 0 { + // Sent nowhere is not delivered: said as the walk's failure, so the operator is not told it landed. + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s (ask %s) was sent to no machine", settingsWalkFor(*s), s.Ask) + } if len(r.Skipped) > 0 { p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ") } @@ -197,14 +228,8 @@ func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) ( } switch verdict { case "": - // What the send carried and broke is put back at once, as a judged send's is; the module whose settings - // changed has no build to put back and is judged to its verdict. - if returned := len(g.Returned); putBackBroken(ctx, open, p, g, "", nil) && len(g.Returned) > returned { - return true, nil - } return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g))) case inventory.GatePassed: - passCarried(ctx, open, p, g, "") r.Done = append(r.Done, node) r.Next++ r.Gate = nil @@ -216,19 +241,12 @@ func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) ( fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil } - p.Note = "" - batched, back := batchingRollbacks(ctx) - failCarried(batched, open, p, g, "") - sendRollbacks(ctx, open, p, back) - putBack := p.Note + p.State = inventory.PlanFailed p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+ "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) if rest := r.Order[r.Next+1:]; len(rest) > 0 { p.Note += "; not sent: " + strings.Join(rest, ", ") } - if putBack != "" { - p.Note += "; " + putBack - } fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil } diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 8cf8e580..0c7e81f9 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -57,8 +57,8 @@ func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) { } } -// The ask says, before the Approve, that the layer is sent at once and judged, and that the operator is told when -// it has landed; neither the ask nor the proposer's answer names a push any more. +// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every +// walk's is; neither the ask nor the proposer's answer names a push any more. func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { for _, c := range []struct { in proposeInput @@ -81,8 +81,8 @@ func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { if err := q.Check(r.now); err != nil { t.Fatalf("the ask is refused: %v", err) } - if !strings.Contains(q.Explanation, "and sent at once to "+c.where+", one machine at a time, each judged; you "+ - "are told when it has landed") { + if !strings.Contains(q.Explanation, "and sent at once to "+c.where+", one machine at a time, each judged, "+ + "every build there kept; its end is told as every walk's is") { t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation) } approve := q.Options[0] @@ -113,6 +113,13 @@ func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { if _, err := assign(ctx, open, "laptop", "notes"); err != nil { t.Fatal(err) } + // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). + if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { + t.Fatal(err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { + t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) + } set := setLayerIn(open) now := time.Now() values := map[string]any{"x": "1"} @@ -154,8 +161,9 @@ func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { } t.Cleanup(func() { openSettingsWalk = was }) again := map[string]any{"x": "2"} - r := settingsProposal{Module: "notes", Node: "laptop", Values: again, From: "g14/claude-code", At: now, - Digest: layerDigest(again), BeforeDigest: layerDigest(nil)} + byHand := map[string]any{"x": "0"} + r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, + From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { t.Errorf("an unopened walk: %v", err) @@ -168,6 +176,10 @@ type walkMesh struct { open *stores sent [][]string broken map[string]bool + // kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move. + kept []bool + moves map[string][]dataMove + heard map[string]bool } func aSettledMesh(t *testing.T) *walkMesh { @@ -175,7 +187,8 @@ func aSettledMesh(t *testing.T) *walkMesh { open := aMesh(t) ctx := t.Context() inv := open.inventory - w := &walkMesh{open: open, broken: map[string]bool{}} + w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, + heard: map[string]bool{"anchor": true, "laptop": true}} withConditionsInMemory(t) was := doctorFrom doctorFrom = nil @@ -199,7 +212,8 @@ func aSettledMesh(t *testing.T) *walkMesh { t.Fatal(err) } } - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts + wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves + settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) if err != nil { @@ -211,12 +225,11 @@ func aSettledMesh(t *testing.T) *walkMesh { } return f.moves(node, modules, sent, known, all), nil } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } n := 0 sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { w.sent = append(w.sent, append([]string(nil), names...)) + w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) current, err := open.inventory.CurrentBuilds(ctx) if err != nil { return nil, err @@ -258,6 +271,7 @@ func aSettledMesh(t *testing.T) *walkMesh { gateSettle, gateEvery = 0, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + settingsDataMoves = wasData gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) return w @@ -325,6 +339,9 @@ func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { ctx := t.Context() inv := w.open.inventory w.broken["anchor"] = true + if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { + t.Fatal(err) + } if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { t.Fatal(err) } @@ -340,6 +357,9 @@ func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) } + if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { + t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) + } // A build that waits for a gate is still walked: the failed settings walk holds nothing back. w.broken["anchor"] = false now := time.Now().Add(-time.Minute) @@ -357,3 +377,88 @@ func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) } } + +// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or +// waiting for a gate stays where it is (ADR 0344 §3). +func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + gateEvery = 0 + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { + t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) + } + // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") + if err != nil || kept["app"] != "c1" { + t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) + } + if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { + t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) + } +} + +// A machine whose send would move a running module's data stops the walk there, failed and said with the push that +// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached +// no machine is not said delivered. +func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + gateEvery = 0 + w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || + !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || + !strings.Contains(p.Note, "push laptop --move app") { + t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) + } + + v := aSettledMesh(t) + v.heard = map[string]bool{"laptop": true} + gateEvery = 0 + if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, v.open) + } + p = v.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || + !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { + t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) + } + + u := aSettledMesh(t) + u.heard = map[string]bool{} + if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, u.open) + if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { + t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) + } +} From e32ca00aed9df8503da861c22984a17cb27038f2 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 16:55:49 +0200 Subject: [PATCH 3/4] =?UTF-8?q?Test=20that=20a=20settings=20send=20moves?= =?UTF-8?q?=20no=20ungated=20build=20waiting=20on=20its=20machine=20(issue?= =?UTF-8?q?=20641,=20ADR=200344=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/settings_walk_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 0c7e81f9..72107bfc 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -413,6 +413,18 @@ func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) } + // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. + f, err := readMoveFacts(ctx, inv) + if err != nil { + t.Fatal(err) + } + waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) + if len(waiting) != 1 || waiting[0].Module != "app" { + t.Fatalf("anchor waits for %+v, want app's c2", waiting) + } + if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { + t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) + } } // A machine whose send would move a running module's data stops the walk there, failed and said with the push that From 8bbb1b19ae63ad0a745176eae7f40f3939d27497 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 17:33:07 +0200 Subject: [PATCH 4/4] A settings gate keeps the module's build, and a walk that cannot keep a build fails (issue 641 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without a build on the gate, a merge walk sending the machine a new build of the module was read as the settings send's own report. A refusal for a last send not known, or a kept build gone, never clears, so retrying it every tick held the walk of waiting builds; it now fails the walk with why. Every failure names the machines passed over as away (ADR 0344 §6). --- cmd/mesh-controller/recorded_kept.go | 13 +++- cmd/mesh-controller/settings_walk.go | 62 ++++++++++++---- cmd/mesh-controller/settings_walk_test.go | 89 +++++++++++++++++++++++ 3 files changed, 145 insertions(+), 19 deletions(-) diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go index 2b65aa4b..bc9560ff 100644 --- a/cmd/mesh-controller/recorded_kept.go +++ b/cmd/mesh-controller/recorded_kept.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "sort" @@ -93,6 +94,11 @@ func keepingBuildsForSettings(ctx context.Context, node string) context.Context return context.WithValue(ctx, keepForSettingsKey{}, node) } +// errSettingsCannotKeep is a settings walk's send refused because a build of its machine cannot be kept: what the +// machine was last sent is not known, or a build it runs is gone from the records. It does not clear by itself, so +// the walk fails on it rather than trying again (ADR 0344). +var errSettingsCannotKeep = errors.New("the settings cannot be sent with every build kept") + // settingsKeptOn is whether this context is a settings walk's send to this machine. func settingsKeptOn(ctx context.Context, node string) bool { on, _ := ctx.Value(keepForSettingsKey{}).(string) @@ -123,8 +129,7 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st } if !known { if settingsKeptOn(ctx, node) { - return nil, fmt.Errorf("what %s was last sent is not known, so its settings cannot be sent there with "+ - "every build kept (novox/hq ADR 0344)", node) + return nil, fmt.Errorf("%w: what %s was last sent is not known (novox/hq ADR 0344)", errSettingsCannotKeep, node) } if every { return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+ @@ -187,8 +192,8 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri return nil, err } if !found && settingsKeptOn(ctx, node) { - return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: its settings cannot "+ - "be sent there with it kept (novox/hq ADR 0344)", node, m, short(kept[m])) + return nil, fmt.Errorf("%w: %s runs %s's build %s, which the build records no longer hold (novox/hq ADR 0344)", + errSettingsCannotKeep, node, m, short(kept[m])) } if !found && everyBuildKept(ctx, node) { return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ diff --git a/cmd/mesh-controller/settings_walk.go b/cmd/mesh-controller/settings_walk.go index cd756b3e..8c5acb3b 100644 --- a/cmd/mesh-controller/settings_walk.go +++ b/cmd/mesh-controller/settings_walk.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "slices" "sort" @@ -179,26 +180,37 @@ func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) ( } // **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before // that machine, failed and said, and `push --move` sends it. - if held, err := settingsDataMoves(ctx, open, node); err != nil { + held, err := settingsDataMoves(ctx, open, node) + if errors.Is(err, errSettingsCannotKeep) { + return failBefore(p, node, err.Error()) + } + if err != nil { return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err) - } else if len(held) > 0 { + } + if len(held) > 0 { var said strings.Builder sayDataHeld(&said, node, held) - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s not sent to %s: a running module's data would move there — %s; the layer "+ - "stays as the operator approved it", settingsWalkFor(*s), node, strings.Join(strings.Fields(said.String()), " ")) - if rest := r.Order[r.Next+1:]; len(rest) > 0 { - p.Note += "; not sent: " + strings.Join(rest, ", ") - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil + return failBefore(p, node, "a running module's data would move there — "+ + strings.Join(strings.Fields(said.String()), " ")) } sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node) + if errors.Is(err, errSettingsCannotKeep) { + // It does not clear by itself: tried again on every tick, the walk would stay open and hold the walk + // of waiting builds until a person came. + return failBefore(p, node, err.Error()) + } if err != nil { return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err) } + // **The module's build is the one the gate judges** (issue 641 review): kept by this send, it is what a + // crossing send — a merge's walk sending the machine a new build of the module — supersedes, so this gate + // never reads that send's report as its own. + kept := "" + if builds, known, err := open.inventory.SentBuilds(ctx, node); err == nil && known { + kept = builds[s.Module] + } r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, - Sent: sentNow(ctx, open.inventory, sent), Grants: grants} + From: kept, To: kept, Sent: sentNow(ctx, open.inventory, sent), Grants: grants} p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node, settingsWalkFor(*s), s.Ask, s.Module) if said := grantsSaid(grants); said != "" { @@ -243,10 +255,30 @@ func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) ( } p.State = inventory.PlanFailed p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+ - "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) - if rest := r.Order[r.Next+1:]; len(rest) > 0 { - p.Note += "; not sent: " + strings.Join(rest, ", ") - } + "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + unreached(r) fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil } + +// failBefore ends a settings walk failed before it sent a machine, saying why, and that the layer stays. +func failBefore(p *inventory.Plan, node, why string) (bool, error) { + r, s := p.Release, p.Release.Settings + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s not sent to %s: %s; the layer stays as the operator approved it", settingsWalkFor(*s), + node, why) + unreached(r) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil +} + +// unreached is what a stopped settings walk left without the layer, in words: the machines after the one it stopped +// at, and those passed over as away (ADR 0344 §6). Empty when there are none. +func unreached(r *inventory.PlanRelease) string { + var out string + if r.Next+1 < len(r.Order) { + out += "; not sent: " + strings.Join(r.Order[r.Next+1:], ", ") + } + if len(r.Skipped) > 0 { + out += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") + } + return out +} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 72107bfc..f8cca40a 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "reflect" "strings" @@ -474,3 +475,91 @@ func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) } } + +// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new +// build of the module while the settings gate judges there — supersedes it, and its report is never read as the +// settings send's (issue 641 review, finding 1). +func TestACrossingSendSupersedesASettingsGate(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, w.open) + p := w.walk(t) + if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { + t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) + } + // Another walk sends anchor app's c2 while the settings gate judges there. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", + Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, w.open) + p = w.walk(t) + if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || + !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { + t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) + } +} + +// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is +// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a +// failure names the machines passed over as away (issue 641 review, findings 2 and 3). +func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. + if _, err := inv.AddNode(ctx, "fresh"); err != nil { + t.Fatal(err) + } + if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { + t.Fatalf("a settings send to a machine never sent: %v", err) + } + w.heard = map[string]bool{"laptop": true} + settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { + return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) + } + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, w.open) + p := w.walk(t) + if p.State != inventory.PlanFailed || len(w.sent) != 0 || + !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || + !strings.Contains(p.Note, "not heard from, left as they were: anchor") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + // It is not open, so a build waiting for a gate is walked. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) + } +}