Files
mesh-controller/cmd/mesh-controller/release_plan.go
T
jschoubben 0a40c046cf plans --what-if: the plan a merge would produce, read before merging
Given a repository and the files a branch changes (or the modules by name), plans answers with the tiers
the merge handler would produce — the modules those files touch, what packages their source, everything
reachable along the dependency relation — and what each tier does: built and sent to its machines, or
built and left because its policy records. Saved nowhere. On the seat as plans {repository, paths|modules}.
2026-10-01 22:38:49 +02:00

743 lines
22 KiB
Go

package main
import (
"context"
"flag"
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A merge produces a tiered plan the mesh keeps (novox/hq ADR 0162).
//
// The handler that hears the merge computes the plan from the catalogue's one dependency relation,
// writes it to the store, asks the first tier and returns — the receive loop is never held by a
// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
// plan where it left it.
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
const planWaitBound = 30 * time.Minute
// tiersOf sorts a set of modules into tiers along the ordering edges among them: tier 0 depends
// on nothing else in the set, tier 1 only on tier 0, and so on. An edge to a module outside the set says
// nothing about the order inside it. A cycle — which the catalogue should never produce — puts
// what remains in one last tier rather than losing it, and is said by the caller.
func tiersOf(set []string, edges []inventory.Edge) [][]string {
in := map[string]bool{}
for _, m := range set {
in[m] = true
}
deps := map[string]map[string]bool{}
for _, m := range set {
deps[m] = map[string]bool{}
}
for _, e := range edges {
// A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's
// build needs nothing of A's first. The other kinds order: stands-on and declared after
// the base is built, built-by after the build machine is built and running — except for
// what the build machine itself stands on. The runtime image is built by the builder and
// the builder is built on the runtime image; the image comes first, built by the builder
// that is running, which is the only one there could be.
if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages {
continue
}
if e.Kind == inventory.EdgeBuiltBy && isBaseOf(e.From, e.To, edges, in) {
continue
}
deps[e.From][e.To] = true
}
placed := map[string]bool{}
var tiers [][]string
for len(placed) < len(set) {
var tier []string
for _, m := range set {
if placed[m] {
continue
}
free := true
for d := range deps[m] {
if !placed[d] {
free = false
break
}
}
if free {
tier = append(tier, m)
}
}
if len(tier) == 0 {
// A cycle: everything left, together, and the caller says so.
for _, m := range set {
if !placed[m] {
tier = append(tier, m)
}
}
}
sort.Strings(tier)
for _, m := range tier {
placed[m] = true
}
tiers = append(tiers, tier)
}
return tiers
}
// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along
// the build edges alone.
func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool {
seen := map[string]bool{}
var walk func(string) bool
walk = func(m string) bool {
if m == base {
return true
}
if seen[m] {
return false
}
seen[m] = true
for _, e := range edges {
if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) {
return true
}
}
return false
}
return walk(to)
}
// reachableFrom is the moved modules plus everything that depends on them, through every layer:
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change.
func reachableFrom(moved []string, edges []inventory.Edge) []string {
in := map[string]bool{}
for _, m := range moved {
in[m] = true
}
for grew := true; grew; {
grew = false
for _, e := range edges {
if e.Kind == inventory.EdgeBuiltBy {
continue
}
if in[e.To] && !in[e.From] {
in[e.From] = true
grew = true
}
}
}
out := make([]string, 0, len(in))
for m := range in {
out = append(out, m)
}
sort.Strings(out)
return out
}
// hasCycle says whether the tiers' last tier holds modules that still depend on each other.
func hasCycle(tiers [][]string, edges []inventory.Edge) bool {
if len(tiers) == 0 {
return false
}
last := map[string]bool{}
for _, m := range tiers[len(tiers)-1] {
last[m] = true
}
for _, e := range edges {
if last[e.From] && last[e.To] {
return true
}
}
return false
}
// planFor is the plan a merge produces: the moved modules and everything reachable from them,
// tiered, with the merge it answers.
func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan {
set := reachableFrom(moved, edges)
tiers := tiersOf(set, edges)
modules := map[string]*inventory.PlanModule{}
for _, name := range set {
modules[name] = &inventory.PlanModule{}
}
return inventory.Plan{
ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()),
Repository: m.Owner + "/" + m.Repo,
Commit: m.Commit,
Created: time.Now().UTC(),
State: inventory.PlanBuilding,
Tiers: tiers,
Modules: modules,
}
}
// gates is what the next tier needs running from this one: a module of the tier that a later
// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
// the machines running it before the next tier is asked. A base an image stands on need only be
// built; a source another module packages need not even be that.
func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) []string {
if p.Tier >= len(p.Tiers) {
return nil
}
inTier := map[string]bool{}
all := map[string]bool{}
for _, tier := range p.Tiers {
for _, m := range tier {
all[m] = true
}
}
for _, m := range p.Tiers[p.Tier] {
inTier[m] = true
}
later := map[string]bool{}
for _, tier := range p.Tiers[p.Tier+1:] {
for _, m := range tier {
later[m] = true
}
}
seen := map[string]bool{}
var out []string
for _, e := range edges {
if later[e.From] && inTier[e.To] && e.Kind == inventory.EdgeBuiltBy && !seen[e.To] && rollsOut(e.To) &&
!isBaseOf(e.From, e.To, edges, all) {
seen[e.To] = true
out = append(out, e.To)
}
}
sort.Strings(out)
return out
}
// applied says whether every machine running the module has reported since the module was built.
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
at := map[string]*time.Time{}
for _, r := range reports {
at[r.Node] = r.At
}
var waiting []string
for _, n := range running {
if t := at[n]; t == nil || t.Before(builtAt) {
waiting = append(waiting, n)
}
}
return len(waiting) == 0, waiting
}
// askTier asks the build machine for every module of the tier, and marks each asked. A module
// the catalogue no longer holds, or whose ask could not be made, is a failure of the plan: a tier
// half asked is a tier that will never complete.
func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error {
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
now := time.Now().UTC()
for _, name := range p.Tiers[p.Tier] {
state := p.Modules[name]
if state == nil {
state = &inventory.PlanModule{}
p.Modules[name] = state
}
e, known := byName[name]
if !known {
state.State = "failed"
state.Why = "no longer in the catalogue"
p.State = inventory.PlanFailed
p.Note = name + " is no longer in the catalogue"
continue
}
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
fmt.Printf(" tier %d: ", p.Tier)
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, 0); err != nil {
state.State = "failed"
state.Why = err.Error()
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
continue
}
state.State = "asked"
state.AskedAt = &now
}
return nil
}
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
// advances what that completes. Called from the daemon's take-in of every outcome.
func planBuilt(ctx context.Context, open *stores, module, commit, failed string) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
fmt.Printf("plans: cannot read them: %v\n", err)
return
}
now := time.Now().UTC()
for i := range plans {
p := &plans[i]
if p.Tier >= len(p.Tiers) {
continue
}
inTier := false
for _, m := range p.Tiers[p.Tier] {
if m == module {
inTier = true
}
}
if !inTier {
continue
}
state := p.Modules[module]
if state == nil {
state = &inventory.PlanModule{}
p.Modules[module] = state
}
if failed != "" {
state.State = "failed"
state.Why = failed
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
} else {
state.State = "built"
state.BuiltAt = &now
state.Commit = commit
}
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
continue
}
if p.State == inventory.PlanFailed {
fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
}
}
advancePlans(ctx, open)
}
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
func advancePlans(ctx context.Context, open *stores) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
fmt.Printf("plans: cannot read them: %v\n", err)
return
}
if len(plans) == 0 {
return
}
edges, err := inv.Dependencies(ctx)
if err != nil {
fmt.Printf("plans: cannot read the dependencies: %v\n", err)
return
}
rollsOut := func(module string) bool {
u, err := inv.UpgradeOf(ctx, module)
return err == nil && u.RollOut
}
for i := range plans {
p := &plans[i]
for p.Open() {
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
if err != nil {
fmt.Printf("%s: %v\n", p.ID, err)
break
}
if err := inv.SavePlan(ctx, *p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
break
}
if !moved {
break
}
}
}
}
// advanceOnce takes one step of one plan and says whether anything changed.
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
inv := open.inventory
if p.Tier >= len(p.Tiers) {
p.State = inventory.PlanDone
fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers))
return true, nil
}
tier := p.Tiers[p.Tier]
// Not yet asked: ask.
unasked := 0
for _, m := range tier {
if s := p.Modules[m]; s == nil || s.State == "" {
unasked++
}
}
if unasked == len(tier) {
if err := askTier(ctx, inv, p); err != nil {
return false, err
}
return true, nil
}
// Asked: wait for every build.
var latest time.Time
for _, m := range tier {
s := p.Modules[m]
if s == nil || s.State != "built" {
return false, nil
}
if s.BuiltAt != nil && s.BuiltAt.After(latest) {
latest = *s.BuiltAt
}
}
// Built: send every module of the tier whose policy rolls out, once, to the machines running
// it — whether or not its source commit moved. A dependent rebuilt because its base moved, or
// a module that packages another repository's source, keeps its commit; the catalogue announces
// no move for it and its machines would keep the old image until somebody pushed (novox/hq
// issue 189). A module whose policy records is built and left, as its policy says.
for _, m := range tier {
state := p.Modules[m]
if state == nil || state.SentAt != nil || !rollsOut(m) {
continue
}
running, err := inv.Running(ctx, m)
if err != nil {
return false, err
}
now := time.Now().UTC()
state.SentAt = &now
if len(running) == 0 {
continue
}
if err := sendTo(ctx, open, running); err != nil {
return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(running, ", "), p.Tier, err)
}
fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(running, ", "))
return true, nil
}
// And wait for what the next tier needs running.
needed := gates(*p, edges, rollsOut)
if len(needed) > 0 {
reports, err := inv.LastReports(ctx)
if err != nil {
return false, err
}
var waiting []string
for _, m := range needed {
running, err := inv.Running(ctx, m)
if err != nil {
return false, err
}
state := p.Modules[m]
if state == nil {
state = &inventory.PlanModule{}
p.Modules[m] = state
}
// The plan sends what it waits for. A rebuild from the same source commit is not a
// move the catalogue announces — the build machine rebuilt for a controller change
// is one — so the roll-out that opens this gate is the plan's to make, once, and
// the reports that open it are the ones after the send.
since := latest
if state.BuiltAt != nil {
since = *state.BuiltAt
}
if state.SentAt != nil && state.SentAt.After(since) {
since = *state.SentAt
}
if ok, on := applied(m, since, running, reports); !ok {
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
}
}
if len(waiting) > 0 {
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied"
changed := p.State != inventory.PlanRolling || p.Note != note
p.State = inventory.PlanRolling
p.Note = note
return changed, nil
}
}
p.Tier++
p.State = inventory.PlanBuilding
p.Note = ""
if p.Tier < len(p.Tiers) {
fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", "))
}
return true, nil
}
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
func planTicker(ctx context.Context, open *stores) {
advancePlans(ctx, open)
tick := time.NewTicker(30 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
advancePlans(ctx, open)
}
}
}
// planLine is one plan as `status` says it.
func planLine(p inventory.Plan, now time.Time) string {
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
switch p.State {
case inventory.PlanDone:
return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
case inventory.PlanFailed:
return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
}
since := now.Sub(p.Updated).Round(time.Minute)
late := ""
if since > planWaitBound {
late = " — LATE"
}
what := "building"
if p.State == inventory.PlanRolling {
what = p.Note
}
return fmt.Sprintf("%s %s %s, %s for %s%s", p.Repository, short(p.Commit), where, what, since, late)
}
// planFailedBuild marks the module a failed build was for when the result names no module: by the
// repository and path the plan's modules were asked at.
func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
return
}
for _, e := range entries {
if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed)
return
}
}
}
func repositoryMatches(a, b string) bool {
trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
}
// planStatus is one plan as `status --json` says it.
type planStatus struct {
ID string `json:"id"`
Repository string `json:"repository"`
Commit string `json:"commit"`
State string `json:"state"`
Tier int `json:"tier"`
Tiers int `json:"tiers"`
Waiting string `json:"waiting,omitempty"`
Since time.Time `json:"since"`
Late bool `json:"late"`
}
func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
out := make([]planStatus, 0, len(plans))
for _, p := range plans {
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated}
if p.Open() {
ps.Waiting = p.Note
if ps.Waiting == "" {
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
}
ps.Late = now.Sub(p.Updated) > planWaitBound
}
out = append(out, ps)
}
return out
}
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
var open []inventory.Plan
late := 0
for _, p := range plans {
if p.Open() {
open = append(open, p)
if time.Since(p.Updated) > planWaitBound {
late++
}
}
}
return open, late
}
// plansCommand says what the last merges produced and where each stands; given an id, one plan
// tier by tier with every module's state.
func plansCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("plans", flag.ContinueOnError)
limit := set.Int("n", 10, "how many to show")
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
modules := set.String("modules", "", "or the modules it would change, comma-separated")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
now := time.Now()
if len(positionals) == 1 {
p, err := inv.PlanByID(ctx, positionals[0])
if err != nil {
return err
}
fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
for i, tier := range p.Tiers {
marker := " "
if i == p.Tier && p.Open() {
marker = ">"
}
fmt.Printf("%s tier %d\n", marker, i)
for _, m := range tier {
s := p.Modules[m]
state := "not yet asked"
if s != nil && s.State != "" {
state = s.State
if s.Commit != "" {
state += " from " + short(s.Commit)
}
if s.Why != "" {
state += ": " + s.Why
}
}
fmt.Printf(" %-22s %s\n", m, state)
}
}
return nil
}
if *whatIf != "" {
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
}
if len(positionals) == 2 && positionals[0] == "stop" {
p, err := inv.PlanByID(ctx, positionals[1])
if err != nil {
return err
}
if !p.Open() {
return fmt.Errorf("%s is already %s", p.ID, p.State)
}
p.State = inventory.PlanFailed
p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
if err := inv.SavePlan(ctx, p); err != nil {
return err
}
fmt.Printf("%s stopped at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
p.ID, p.Tier, len(p.Tiers))
return nil
}
plans, err := inv.RecentPlans(ctx, *limit)
if err != nil {
return err
}
if len(plans) == 0 {
fmt.Println("no merge has produced a plan yet")
return nil
}
for _, p := range plans {
fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
}
return nil
}
// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one
// and saved nowhere: the modules the repository's changed files touch (or the modules named), what
// packages their source, everything reachable from them, in tiers. For reading before merging.
func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string) error {
owner, repo, found := strings.Cut(repository, "/")
if !found {
return fmt.Errorf("--what-if takes owner/repository, not %q", repository)
}
m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "what-if", Paths: paths}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return err
}
var from, packaging []inventory.Entry
named := map[string]bool{}
for _, name := range modules {
named[name] = true
}
for _, e := range entries {
switch {
case named[e.Manifest.Module]:
from = append(from, e)
case len(named) == 0 && sourceIs(e.Source, m):
from = append(from, e)
case readsFrom(read[e.Manifest.Module], m):
packaging = append(packaging, e)
}
}
if len(named) == 0 {
from = whatTheMergeTouched(from, entries, m)
}
moved := append(append([]inventory.Entry{}, from...), packaging...)
if len(moved) == 0 {
fmt.Printf("a merge of %s changing %s would build nothing the mesh holds\n", repository,
orNone(strings.Join(append(paths, modules...), ", ")))
return nil
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return err
}
var names []string
for _, e := range moved {
names = append(names, e.Manifest.Module)
}
p := planOfMerge(m, names, edges)
fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers))
rolls := map[string]string{}
for i, tier := range p.Tiers {
fmt.Printf(" tier %d\n", i)
for _, name := range tier {
how := "built; its policy records, so nothing is sent"
if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
running, _ := inv.Running(ctx, name)
how = "built, then sent to " + orNone(strings.Join(running, ", "))
rolls[name] = how
}
fmt.Printf(" %-22s %s\n", name, how)
}
}
if hasCycle(p.Tiers, edges) {
fmt.Println(" the last tier depends on itself and would be built together, in no order")
}
if len(packaging) > 0 {
var also []string
for _, e := range packaging {
also = append(also, e.Manifest.Module)
}
fmt.Printf(" %s package source from %s, so they are rebuilt without their own source moving\n",
strings.Join(also, ", "), repository)
}
return nil
}
func splitList(s string) []string {
var out []string
for _, part := range strings.Split(s, ",") {
if part = strings.TrimSpace(part); part != "" {
out = append(out, part)
}
}
return out
}