diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go new file mode 100644 index 00000000..cae82843 --- /dev/null +++ b/cmd/mesh-controller/core_downgrade.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// **No send but a person's `push ` and a gate's own put-back moves the mesh's own core to a build older +// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a +// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under +// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans; +// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then +// sent the control node the controller build before it. The older binary read the row it could not serve and the +// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR +// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here, +// without needing its trigger: whatever made the mesh's record older, the send keeps the build. +// +// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last +// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or +// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend +// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by +// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until +// a send carries the record — a person's `push ` accepts the older build — or the record moves forward: a +// newer merge, or `rebuild mesh-controller`. +// +// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits' +// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's +// current build by the same order, and the guard does not stop that move. A build that failed its gate is never +// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own +// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back +// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's +// schema already (issue 352). + +// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record. +const kindCoreDowngrade = "core-downgrade-kept" + +// coreDowngradeSource is the condition's source: the sends, which raise and clear it. +const coreDowngradeSource = "sends" + +// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the +// store and its seat's row, and the node-engine, which reads what the controller declares. +var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} + +// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate. +type askedBuild struct { + asked time.Time + failedGate bool +} + +// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when +// the request time is not known), and whether it failed its gate — and false when the records hold none. A +// variable, so a test reads its own order. +var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) { + builds, err := inv.Builds(ctx, module, askedLookBack) + if err != nil { + return askedBuild{}, false, err + } + for _, b := range builds { + if b.Worked() && sameCommit(b.Commit, commit) { + failed, err := inv.GateFailed(ctx, b.ID) + if err != nil { + return askedBuild{}, false, err + } + return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil + } + } + return askedBuild{}, false, nil +} + +// askedLookBack is how many of a module's newest builds are read for when a build was asked. +const askedLookBack = 50 + +// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record. +type coreDowngrade struct { + module, node, component string + // sent is the build the machine was last sent, record the one the mesh holds. + sent, record string + sentAsked, recordAsked time.Time +} + +func (d coreDowngrade) String() string { + return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+ + "keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+ + "(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record), + d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node, + d.module) +} + +// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine +// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move +// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure +// but for asked. +func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope, + asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) { + var out []coreDowngrade + for module, was := range sent { + component := coreComponent(module) + now := record[module] + if !guarded(component) || was == "" || now == "" || sameCommit(was, now) { + continue + } + if scope.puttingBack && scope.modules[module] { + continue + } + from, known, err := asked(module, was) + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err) + } + if !known || from.failedGate { + continue + } + to, known, err := asked(module, now) + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err) + } + if known && to.asked.Before(from.asked) { + out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now, + sentAsked: from.asked, recordAsked: to.asked}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module }) + return out, nil +} + +func guarded(component string) bool { + for _, c := range downgradeGuarded { + if c == component { + return true + } + } + return false +} + +// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it +// was last sent, module → commit, each said and raised as a condition. +func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string, + current map[string]inventory.CurrentBuild) (map[string]string, error) { + record := map[string]string{} + for m, c := range current { + record[m] = c.Commit + } + found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) { + return buildAsked(ctx, inv, module, commit) + }) + if err != nil || len(found) == 0 { + return nil, err + } + out := map[string]string{} + for _, d := range found { + out[d.module] = d.sent + sayCoreDowngrade(ctx, d) + } + return out, nil +} + +// coreDowngradeObservation is a kept core build as a condition: `core...`. +func coreDowngradeObservation(component, node, summary string) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, + Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, + Summary: summary} +} + +func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { + fmt.Printf("KEPT: %s\n", d) + keeper := conditionsFrom + if keeper == nil { + return + } + if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil { + fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err) + } +} + +// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the +// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record +// moved forward — or did not carry at all. A send that kept the newer build leaves it open. +func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) { + keeper := conditionsFrom + if keeper == nil || carried == nil { + return + } + var open []string + for _, c := range downgradeGuarded { + if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found { + open = append(open, c) + } + } + if len(open) == 0 { + return + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err) + return + } + for module, now := range current { + component := coreComponent(module) + if !guarded(component) { + continue + } + if was, has := carried[module]; has && !sameCommit(was, now.Commit) { + continue + } + key := coreDowngradeObservation(component, node, "").Key() + if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil { + fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + } + } +} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index b4dd98e2..f5aa1689 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go new file mode 100644 index 00000000..2e9c5c91 --- /dev/null +++ b/cmd/mesh-controller/issue201_test.go @@ -0,0 +1,259 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, +// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for +// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back +// behind the build its machine was last sent: the send keeps it there, and carries the rest. + +// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed +// their gate. +func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) { + return func(module, commit string) (askedBuild, bool, error) { + at, known := table[module+"@"+commit] + b := askedBuild{asked: at} + for _, f := range failed { + if f == module+"@"+commit { + b.failedGate = true + } + } + return b, known, nil + } +} + +func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { + old, now := t0, t0.Add(time.Minute) + table := map[string]time.Time{ + "mesh-controller@c1": old, "mesh-controller@c2": now, + "mesh-host@h1": old, "mesh-host@h2": now, + "app@a1": old, "app@a2": now, + } + sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"} + record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"} + found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table)) + if err != nil { + t.Fatal(err) + } + if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController || + found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine { + t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found) + } + if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") || + !strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") { + t.Fatalf("it says the machine, the build kept and the two ways out: %s", said) + } + + // The record forward, the same build, or a build the records do not hold: nothing kept. + for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} { + found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]}, + map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table)) + if err != nil || len(found) != 0 { + t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err) + } + } + + // A sent build that failed its gate is never ahead: the record put back before it is sent. + if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 || + found[0].module != "mesh-host" { + t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found) + } + + // A gate's own put-back carries its own modules back, and nothing else. + putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} + if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" { + t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) + } + if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}}, + askedIn(table)); len(found) != 2 { + t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) + } +} + +// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the +// controller is the build before it, c1, which passed its gate when it rolled out. +func core201(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + record := func(commit string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) + b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, + Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + return b + } + before := record("c1aaaaaaaaaa", old) + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } + record("c2bbbbbbbbbb", now) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { + t.Fatal(err) + } + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + wasMoves := machineMoves + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + t.Cleanup(func() { machineMoves = wasMoves }) + return open +} + +// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is +// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer +// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, +// as a person's push does, clears the condition. +func TestReplay201(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + + judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) + if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) + } + keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") + if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { + t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) + } + shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} + if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { + t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) + } + key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() + if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { + t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) + } + + putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) + if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) + } + + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) + if _, found, _ := k.Get(ctx, key); !found { + t.Fatal("a send that kept the newer controller cleared the condition") + } + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) + if _, found, _ := k.Get(ctx, key); found { + t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") + } +} + +// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was +// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that +// failed its gate is never kept. +func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", + Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, + Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { + t.Fatal(err) + } + healer := withScope(ctx, sendScope{}) + if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("the healer's send is made: %v %v", kept, err) + } + if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) + } + if open, _ := k.Open(ctx); len(open) != 0 { + t.Fatalf("a completed put-back raised a condition: %+v", open) + } +} + +// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept +// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own +// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. +func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { + open := core201(t) + ctx := t.Context() + withConditionsInMemory(t) + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + for _, b := range []struct { + commit string + at time.Time + }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, + Asked: b.at}); err != nil { + t.Fatal(err) + } + } + if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { + t.Fatal(err) + } + // The send as it is made: the controller kept at c2, the app moved to the record. + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) + } + t.Cleanup(func() { sendRollout = wasSend }) + + moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) + if err != nil || len(sent) != 1 { + t.Fatalf("the gated send is made: %v %v", sent, err) + } + if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { + t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) + } + builds, _, err := inv.SentBuilds(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + g := &inventory.PlanGate{Carried: moves} + f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} + for _, m := range []string{"app", "mesh-controller"} { + if why, superseded := f.supersededOn(m, "anchor"); superseded { + t.Fatalf("the gate is superseded by its own send: %s", why) + } + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 7cd4f3b6..a5f705b1 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{ "register it, so it was not made.", module, m), Resolved: "Resolved: " + module + " on " + m + " is settled"} }), + // The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq + // issue 201). + kindCoreDowngrade: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + component := idPart(o, 0) + if o.Scope != conditions.ScopeCore || !guarded(component) { + component = "core component" + } + return words{Headline: m + " keeps a newer " + component + " than the mesh holds", + Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m), + Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+ + "the mesh's sends keep the newer one there rather than put it back.", component, m), + Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"} + }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 8aef75c2..7ff94186 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1027,6 +1027,9 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri if err != nil { return "", err } + // A send that carried the mesh's record of the controller and the node-engine — a person's push included — + // ends the condition of one kept over it (novox/hq issue 201). + settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds) if len(s.declared.Bindings) > 0 { // And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232): // what the next resolution keeps it at. On the same outliving context as the send's record. diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go index a8e8cf5b..ecf9e25c 100644 --- a/cmd/mesh-controller/recorded_kept.go +++ b/cmd/mesh-controller/recorded_kept.go @@ -135,6 +135,17 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st } out[m] = was } + // **And the controller and the node-engine at the build the machine was last sent, over an older record** + // (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already. + if !every { + core, err := coreKept(ctx, inv, node, sent, current) + if err != nil { + return nil, err + } + for m, was := range core { + out[m] = was + } + } return out, nil } @@ -159,6 +170,12 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ "list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m])) } + if !found && guarded(coreComponent(m)) { + return nil, fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+ + "no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+ + "a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)", + node, m, short(kept[m]), node, m) + } if !found { return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+ "records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+ diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index bc16fd84..a97cf4a7 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -51,6 +51,9 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool + // puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build + // older than the one its machine runs (novox/hq issue 201). + puttingBack bool } type sendScopeKey struct{} @@ -237,6 +240,32 @@ var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node str return f.moves(node, modules, sent, known, all), nil } +// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does +// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the +// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that +// call this), so a move it could not keep is still counted. +func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) { + if len(moves) == 0 { + return moves, nil + } + keeping := ctx + if _, on := keptExcept(ctx); !on { + keeping = keepingRecorded(ctx) + } + keeps, err := recordedKept(keeping, open, node) + if err != nil || len(keeps) == 0 { + return moves, err + } + var out []inventory.CarriedMove + for _, mv := range moves { + if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { + continue + } + out = append(out, mv) + } + return out, nil +} + // ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the // machines named, and why; the machine holding the bus, added only for its user list, is left instead. func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) { @@ -254,19 +283,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st if err != nil { return nil, err } - // The grants step composes every module at the build its machine was last sent (novox/hq issue 490): - // a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here. - var keeps map[string]string - if everyBuildKept(ctx, n) { - if keeps, err = recordedKept(ctx, open, n); err != nil { - return nil, err - } + if moves, err = withoutKept(ctx, open, n, moves); err != nil { + return nil, err } var waiting []string for _, mv := range moves { - if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { - continue - } if !scope.judged[n] && !scope.modules[mv.Module] { waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) } @@ -308,6 +329,11 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. if err != nil { return nil, nil, nil, err } + // A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module + // kept over an older record judged as moving back to it would read the gate as superseded by its own send. + if moves, err = withoutKept(ctx, open, node, moves); err != nil { + return nil, nil, nil, err + } own := func(module string) bool { return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) } diff --git a/testdata/store-tests b/testdata/store-tests index 124e9a9a..7586e2d0 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -31,4 +31,7 @@ ./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb ./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave ./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave +./cmd/mesh-controller TestReplay201 +./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend +./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore ./cmd/mesh-controller TestReplay401