package main import ( "context" "fmt" "sort" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) // **No send but a person's `push ` and a gate's own put-back moves the mesh's own core to a build older // than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a // plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under // sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans; // the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then // sent the control node the controller build before it. The older binary read the row it could not serve and the // mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR // 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here, // without needing its trigger: whatever made the mesh's record older, the send keeps the build. // // **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last // sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or // walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend // leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by // the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until // a send carries the record — a person's `push ` accepts the older build — or the record moves forward: a // newer merge, or `rebuild mesh-controller`. // // Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits' // dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's // current build by the same order, and the guard does not stop that move. A build that failed its gate is never // ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own // send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back // carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's // schema already (issue 352). // kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record. const kindCoreDowngrade = "core-downgrade-kept" // coreDowngradeSource is the condition's source: the sends, which raise and clear it. const coreDowngradeSource = "sends" // downgradeGuarded is the core components a send may not move backwards: the controller, which reads the // store and its seat's row, and the node-engine, which reads what the controller declares. var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} // askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate. type askedBuild struct { asked time.Time failedGate bool } // buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when // the request time is not known), and whether it failed its gate — and false when the records hold none. A // variable, so a test reads its own order. var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) { builds, err := inv.Builds(ctx, module, askedLookBack) if err != nil { return askedBuild{}, false, err } for _, b := range builds { if b.Worked() && sameCommit(b.Commit, commit) { failed, err := inv.GateFailed(ctx, b.ID) if err != nil { return askedBuild{}, false, err } return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil } } return askedBuild{}, false, nil } // askedLookBack is how many of a module's newest builds are read for when a build was asked. const askedLookBack = 50 // coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record. type coreDowngrade struct { module, node, component string // sent is the build the machine was last sent, record the one the mesh holds. sent, record string sentAsked, recordAsked time.Time } func (d coreDowngrade) String() string { return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+ "keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+ "(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record), d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node, d.module) } // coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine // was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move // whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure // but for asked. func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope, asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) { var out []coreDowngrade for module, was := range sent { component := coreComponent(module) now := record[module] if !guarded(component) || was == "" || now == "" || sameCommit(was, now) { continue } if scope.puttingBack && scope.modules[module] { continue } from, known, err := asked(module, was) if err != nil { return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err) } if !known || from.failedGate { continue } to, known, err := asked(module, now) if err != nil { return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err) } if known && to.asked.Before(from.asked) { out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now, sentAsked: from.asked, recordAsked: to.asked}) } } sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module }) return out, nil } func guarded(component string) bool { for _, c := range downgradeGuarded { if c == component { return true } } return false } // coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it // was last sent, module → commit, each said and raised as a condition. func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string, current map[string]inventory.CurrentBuild) (map[string]string, error) { record := map[string]string{} for m, c := range current { record[m] = c.Commit } found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) { return buildAsked(ctx, inv, module, commit) }) if err != nil || len(found) == 0 { return nil, err } out := map[string]string{} for _, d := range found { out[d.module] = d.sent sayCoreDowngrade(ctx, d) } return out, nil } // coreDowngradeObservation is a kept core build as a condition: `core...`. func coreDowngradeObservation(component, node, summary string) conditions.Observation { return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, Summary: summary} } func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { fmt.Printf("KEPT: %s\n", d) keeper := conditionsFrom if keeper == nil { return } if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil { fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err) } } // settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the // send carried at the mesh's record — a person's push that accepted the older build, or a send after the record // moved forward — or did not carry at all. A send that kept the newer build leaves it open. func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) { keeper := conditionsFrom if keeper == nil || carried == nil { return } var open []string for _, c := range downgradeGuarded { if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found { open = append(open, c) } } if len(open) == 0 { return } current, err := inv.CurrentBuilds(ctx) if err != nil { fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err) return } for module, now := range current { component := coreComponent(module) if !guarded(component) { continue } if was, has := carried[module]; has && !sameCommit(was, now.Commit) { continue } key := coreDowngradeObservation(component, node, "").Key() if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil { fmt.Printf("the condition %s could not be cleared: %v\n", key, err) } } }