novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 6 Releases Wiki Activity

Gate a release plan's first machine and roll a failed build back there (hq ADR 0236)

Browse Source 
A build that reported applied was sent everywhere; one that then did nothing, served
no tools or broke its machine's word reached every machine. Now the first machine is
judged by the component's health (the core's definitions, as doctor probes H-*, or a
module's own) three times over two minutes within ten; a failing gate puts the previous
build back there once, marks the build, and says it as a condition and an event.
Upgrades roll out by default; the bus is a planned step; a module deleted at its
source is not built (the public-acme plan failure).
This commit is contained in:
jochen
2026-10-06 18:56:54 +02:00
parent 81f497e5dd
commit d9289ef6d4
41 changed files with 3297 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/release_plan.go
+98 -8
View File
@@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"time"
@@ -223,6 +224,9 @@ func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(str
}
var took []string
for name, s := range old.Modules {
if s != nil && s.State == planDeleted {
continue // deleted at its source: nothing to plan again
}
if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) {
folded[name] = true
took = append(took, name)
@@ -427,7 +431,11 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
} else if askedBefore(asked, state.AskedAt) {
continue
}
if failed != "" {
if failed != "" && deletedAtSource(failed) {
// Deleted at its source by the merge, not broken (novox/hq ADR 0235): the plan goes on.
state.State, state.Why = planDeleted, "deleted at its source: "+firstLine(failed)
forgetDeleted(ctx, inv, module, p)
} else if failed != "" {
state.State = "failed"
state.Why = failed
p.State = inventory.PlanFailed
@@ -575,7 +583,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
var latest time.Time
for _, m := range tier {
s := p.Modules[m]
if s == nil || s.State != "built" {
if s == nil || (s.State != "built" && s.State != planDeleted) {
return false, nil
}
if s.BuiltAt != nil && s.BuiltAt.After(latest) {
@@ -598,7 +606,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
var pending []string
for _, m := range tier {
state := p.Modules[m]
if state == nil || state.SentAt != nil || !rollsOut(m) {
if state == nil || state.SentAt != nil || state.State == planDeleted || !rollsOut(m) {
continue
}
running, err := inv.Running(ctx, m)
@@ -620,29 +628,66 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
switch {
case step.failed != "":
state.Why = step.failed
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s stopped at its first machine in tier %d: %s; %s left as it was",
m, p.Tier, step.failed, orNone(strings.Join(step.rest, ", ")))
// The first machine refused or failed what it was sent, or never said: the gate failed, and
// the build is put back there (novox/hq ADR 0235); the rest are left as they were.
gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed)
p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", ")))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
case step.waiting != "":
pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04")))
continue
}
// **The gate** (novox/hq ADR 0235, to-be 45 §8): the first machine reported the build applied;
// it is judged by its health before anything else is sent — the rest, or, where it is the only
// machine, the plan's next step.
if !policy.Together && state.FirstAt != nil && len(state.First) > 0 {
verdict, err := judgeGate(ctx, open, p, m, state, running, now)
if err != nil {
return false, err
}
switch verdict {
case "":
pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m,
strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate)))
continue
case inventory.GateFailed:
gateFailed(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why)
p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", ")))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
case inventory.GatePassed:
if !state.Gate.Kept {
gatePassed(ctx, open, p, m, state)
state.Gate.Kept = true
}
}
}
switch {
case len(step.send) == 0:
// No machine runs it: nothing to send, and nothing to wait for.
state.SentAt = &now
continue
}
// Read before the send, which records what it carries.
var before map[string]string
var beforeKnown bool
if step.first {
before, beforeKnown, _ = inv.SentBuilds(ctx, step.send[0])
}
// What sendToEach answers, not what was asked: the machine holding the bus is sent before
// the first when its user list must change (issue 249), and the plan waits for it too.
sent, err := sendToEach(ctx, open, step.send)
sent, err := sendRollout(ctx, open, step.send)
if err != nil {
// Not marked sent, so the next step tries again (issue 249): a grant that could not be
// issued is a send that did not happen.
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err)
}
if step.first {
// What the first machine ran before: what a failed gate puts back (ADR 0235).
if beforeKnown {
state.Previous = before[m]
}
state.First = sent
state.FirstAt = &now
p.State = inventory.PlanRolling
@@ -681,6 +726,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
state = &inventory.PlanModule{}
p.Modules[m] = state
}
if state.State == planDeleted {
continue
}
// The plan sends what it waits for. A rebuild from the same source commit is not a
// move the catalogue announces — the build machine rebuilt for a controller change
// is one — so the roll-out that opens this gate is the plan's to make, once, and
@@ -1033,6 +1081,11 @@ func plansCommand(ctx context.Context, args []string) error {
}
}
fmt.Printf(" %-22s %s\n", m, state)
// The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to,
// verdict, time to it, rolled back or not.
if s != nil && s.Gate != nil {
fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate))
}
}
}
return nil
@@ -1231,6 +1284,12 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
continue
}
at := outcome.At
if !outcome.Worked() && deletedAtSource(outcome.Failed) {
s.State, s.Why = planDeleted, "deleted at its source: "+firstLine(outcome.Failed)
fmt.Printf("%s: %s was deleted at its source (%s): not built, and the plan goes on\n", p.ID, m, outcome.ID)
changed = true
continue
}
if outcome.Worked() {
s.State = "built"
s.BuiltAt = &at
@@ -1252,3 +1311,34 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
func askedBefore(asked time.Time, planAsked *time.Time) bool {
return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked)
}
// firstRunning is the machines sent first that run the module — not the machine holding the bus, sent
// with them only for its user list.
func firstRunning(first, running []string) []string {
var out []string
for _, n := range first {
if slices.Contains(running, n) {
out = append(out, n)
}
}
if len(out) == 0 {
return first
}
return out
}
// planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no
// failure of the plan (novox/hq ADR 0235).
const planDeleted = "deleted"
// forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says
// it otherwise.
func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string, p *inventory.Plan) {
switch err := inv.ForgetModule(ctx, module); {
case err == nil:
fmt.Printf("%s: %s was deleted at its source: forgotten, and the plan goes on\n", p.ID, module)
default:
fmt.Printf("%s: %s was deleted at its source and is not built; it is kept until a person unassigns it and "+
"`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error()))
}
}