novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 9 Releases Wiki Activity

Stop a rollout whose first machine is silent, and choose one that is heard from (hq issue 249, ADR 0218)

Browse Source 
A first machine that does not report within the bound now stops the
module's rollout, naming it. The first machine is the first by name heard
from lately; reports are judged by what the store says was last sent. A
plan that ends says what it built and never sent, and a failed send's
error is kept in the plan's note.
This commit is contained in:
jochen
2026-10-05 18:17:52 +02:00
parent ed90771382
commit 16c5e78fa8
2 changed files with 140 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/release_plan.go
+69 -21
View File
@@ -391,6 +391,10 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
state.Why = failed
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
sayUnsent(p, func(m string) bool {
u, err := inv.UpgradeOf(ctx, m)
return err == nil && u.RollOut
})
} else {
state.State = "built"
state.BuiltAt = &now
@@ -452,8 +456,17 @@ func advanceHeld(ctx context.Context, open *stores) {
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
if err != nil {
fmt.Printf("%s: %v\n", p.ID, err)
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
// the state is left as it was and the step is tried again on the next tick.
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
}
break
}
if p.State == inventory.PlanFailed {
sayUnsent(p, rollsOut)
}
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
break
@@ -545,13 +558,14 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
return false, err
}
var reports []inventory.Reported
if state.FirstAt != nil && !policy.Together {
if !policy.Together {
// Read for the choice of the first machine as well as for its report.
if reports, err = inv.LastReports(ctx); err != nil {
return false, err
}
}
step := nextRollout(*state, running, policy.Together, reports)
now := time.Now().UTC()
step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
switch {
case step.failed != "":
state.Why = step.failed
@@ -561,11 +575,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
case step.waiting != "":
wait := fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Format("15:04"))
if now.Sub(*state.FirstAt) > planWaitBound {
wait += " — LATE"
}
pending = append(pending, wait)
pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Format("15:04")))
continue
case len(step.send) == 0:
// No machine runs it: nothing to send, and nothing to wait for.
@@ -665,22 +675,36 @@ type rolloutStep struct {
// nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218).
//
// Together, every machine running it at once, as the policy says. Otherwise one machine first —
// the first by name, so the choice is the same on every controller and every resume — and the rest
// once each machine the first send reached has reported, after that send, that it applied the
// declaration it was last sent. A report after the send that failed or refused it is the rollout's
// end: the rest are not sent. A machine that has not reported since is waited for; how long it has
// been is the plan's to say.
func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
// Together, every machine running it at once, as the policy says. Otherwise one machine first — the
// first by name among those that have reported within the bound, so a laptop that is away is not
// the one the rest wait on; the first by name when none has; the same choice on every controller and
// every resume — and the rest once each machine the first send reached reports, about the
// declaration it was last sent, that it applied it. Compared by the store's own record of what was
// sent (Reported.Current), never by this controller's clock against the machine's.
//
// **A first machine that fails, refuses, or does not report within the bound stops the rollout
// there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a
// rollout: it held the plan open for ever, read as work in progress (issue 254).
func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported,
now time.Time, bound time.Duration) rolloutStep {
if len(running) == 0 {
return rolloutStep{}
}
if together {
return rolloutStep{send: running}
}
byNode := map[string]inventory.Reported{}
for _, r := range reports {
byNode[r.Node] = r
}
if s.FirstAt == nil {
sorted := append([]string{}, running...)
sort.Strings(sorted)
for _, n := range sorted {
if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound {
return rolloutStep{send: []string{n}, first: true}
}
}
return rolloutStep{send: sorted[:1], first: true}
}
sentFirst := map[string]bool{}
@@ -693,15 +717,11 @@ func nextRollout(s inventory.PlanModule, running []string, together bool, report
rest = append(rest, n)
}
}
byNode := map[string]inventory.Reported{}
for _, r := range reports {
byNode[r.Node] = r
}
var waiting, failed []string
for _, n := range s.First {
r, said := byNode[n]
// Only a report after the send, about what it was last sent, says anything about this build.
if !said || r.At == nil || r.At.Before(*s.FirstAt) || !r.Current {
// Only a report about what it was last sent says anything about this build.
if !said || r.At == nil || !r.Current {
waiting = append(waiting, n)
continue
}
@@ -717,11 +737,34 @@ func nextRollout(s inventory.PlanModule, running []string, together bool, report
return rolloutStep{failed: strings.Join(failed, "; "), rest: rest}
}
if len(waiting) > 0 {
if now.Sub(*s.FirstAt) > bound {
return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s",
strings.Join(waiting, ", "), bound), rest: rest}
}
return rolloutStep{waiting: strings.Join(waiting, ", ")}
}
return rolloutStep{send: rest}
}
// sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249).
// An announced move of a module a plan held was left to that plan; a plan that ends without
// sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing
// saying so. A module whose rollout stopped at its first machine is not among them: that stop was
// the point. Said once.
func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
var unsent []string
for name, s := range p.Modules {
if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) {
unsent = append(unsent, name)
}
}
if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") {
return
}
sort.Strings(unsent)
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
}
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
func planTicker(ctx context.Context, open *stores) {
advancePlans(ctx, open)
@@ -899,6 +942,10 @@ func plansCommand(ctx context.Context, args []string) error {
}
p.State = inventory.PlanFailed
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
sayUnsent(&p, func(m string) bool {
u, err := inv.UpgradeOf(ctx, m)
return err == nil && u.RollOut
})
if err := inv.SavePlan(ctx, p); err != nil {
return err
}
@@ -978,9 +1025,10 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
how := "built; its policy records, so nothing is sent"
if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
running, _ := inv.Running(ctx, name)
reports, _ := inv.LastReports(ctx)
how = "built, then sent to " + orNone(strings.Join(running, ", "))
// One machine first unless the policy says together (novox/hq issue 249).
if first := nextRollout(inventory.PlanModule{}, running, u.Together, nil); first.first && len(running) > 1 {
if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 {
how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it",
first.send[0])
}