jochen
f0fd7fe886
mesh/merge-gate head f0fd7fe8868e merged into trunk a681227af373 — fail: builds mesh-controller → novox; no bus step; head f0fd7fe8868e does not merge clea…
mesh/repo-check head f0fd7fe8868e merged into trunk a681227af373 — fail: head f0fd7fe8868e does not merge cleanly into trunk a681227af373: in conflict test…
mesh/delivery superseded: a newer head of the same pull request
gatedSend took its judged moves from the record, so on a machine kept at a newer controller the gate judged the controller moving back to the record, read its own send's kept controller as a later build there, and was superseded with nothing judged. The kept moves are now left out there as in ungatedIn (withoutKept). The doc says the bus step keeps too, and the three store tests are listed.
216 lines
9.5 KiB
Go
216 lines
9.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
)
|
|
|
|
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
|
|
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
|
|
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
|
|
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
|
|
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
|
|
// sent the control node the controller build before it. The older binary read the row it could not serve and the
|
|
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
|
|
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
|
|
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
|
|
//
|
|
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
|
|
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
|
|
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
|
|
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
|
|
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
|
|
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
|
|
// newer merge, or `rebuild mesh-controller`.
|
|
//
|
|
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
|
|
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
|
|
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
|
|
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
|
|
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
|
|
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
|
|
// schema already (issue 352).
|
|
|
|
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
|
|
const kindCoreDowngrade = "core-downgrade-kept"
|
|
|
|
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
|
|
const coreDowngradeSource = "sends"
|
|
|
|
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
|
|
// store and its seat's row, and the node-engine, which reads what the controller declares.
|
|
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
|
|
|
|
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
|
|
type askedBuild struct {
|
|
asked time.Time
|
|
failedGate bool
|
|
}
|
|
|
|
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
|
|
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
|
|
// variable, so a test reads its own order.
|
|
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
|
|
builds, err := inv.Builds(ctx, module, askedLookBack)
|
|
if err != nil {
|
|
return askedBuild{}, false, err
|
|
}
|
|
for _, b := range builds {
|
|
if b.Worked() && sameCommit(b.Commit, commit) {
|
|
failed, err := inv.GateFailed(ctx, b.ID)
|
|
if err != nil {
|
|
return askedBuild{}, false, err
|
|
}
|
|
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
|
|
}
|
|
}
|
|
return askedBuild{}, false, nil
|
|
}
|
|
|
|
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
|
|
const askedLookBack = 50
|
|
|
|
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
|
|
type coreDowngrade struct {
|
|
module, node, component string
|
|
// sent is the build the machine was last sent, record the one the mesh holds.
|
|
sent, record string
|
|
sentAsked, recordAsked time.Time
|
|
}
|
|
|
|
func (d coreDowngrade) String() string {
|
|
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
|
|
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
|
|
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
|
|
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
|
|
d.module)
|
|
}
|
|
|
|
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
|
|
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
|
|
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
|
|
// but for asked.
|
|
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
|
|
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
|
|
var out []coreDowngrade
|
|
for module, was := range sent {
|
|
component := coreComponent(module)
|
|
now := record[module]
|
|
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
|
|
continue
|
|
}
|
|
if scope.puttingBack && scope.modules[module] {
|
|
continue
|
|
}
|
|
from, known, err := asked(module, was)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
|
|
}
|
|
if !known || from.failedGate {
|
|
continue
|
|
}
|
|
to, known, err := asked(module, now)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
|
|
}
|
|
if known && to.asked.Before(from.asked) {
|
|
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
|
|
sentAsked: from.asked, recordAsked: to.asked})
|
|
}
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
|
|
return out, nil
|
|
}
|
|
|
|
func guarded(component string) bool {
|
|
for _, c := range downgradeGuarded {
|
|
if c == component {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
|
|
// was last sent, module → commit, each said and raised as a condition.
|
|
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
|
|
current map[string]inventory.CurrentBuild) (map[string]string, error) {
|
|
record := map[string]string{}
|
|
for m, c := range current {
|
|
record[m] = c.Commit
|
|
}
|
|
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
|
|
return buildAsked(ctx, inv, module, commit)
|
|
})
|
|
if err != nil || len(found) == 0 {
|
|
return nil, err
|
|
}
|
|
out := map[string]string{}
|
|
for _, d := range found {
|
|
out[d.module] = d.sent
|
|
sayCoreDowngrade(ctx, d)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
|
|
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
|
|
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
|
|
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
|
|
Summary: summary}
|
|
}
|
|
|
|
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
|
|
fmt.Printf("KEPT: %s\n", d)
|
|
keeper := conditionsFrom
|
|
if keeper == nil {
|
|
return
|
|
}
|
|
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
|
|
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
|
|
}
|
|
}
|
|
|
|
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
|
|
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
|
|
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
|
|
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
|
|
keeper := conditionsFrom
|
|
if keeper == nil || carried == nil {
|
|
return
|
|
}
|
|
var open []string
|
|
for _, c := range downgradeGuarded {
|
|
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
|
|
open = append(open, c)
|
|
}
|
|
}
|
|
if len(open) == 0 {
|
|
return
|
|
}
|
|
current, err := inv.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
|
|
return
|
|
}
|
|
for module, now := range current {
|
|
component := coreComponent(module)
|
|
if !guarded(component) {
|
|
continue
|
|
}
|
|
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
|
|
continue
|
|
}
|
|
key := coreDowngradeObservation(component, node, "").Key()
|
|
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
|
|
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
|
}
|
|
}
|
|
}
|