mesh-controller/cmd/mesh-controller/upgrades.go

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"github.com/novox/mesh-controller/internal/gosource"
	"regexp"
	"slices"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// following acts on what the catalogue announces.
//
// **It holds the stores, not a copy of the decision.** What to do about an upgrade is read when
// one arrives, so changing it takes effect on the next upgrade rather than on the next restart of
// the control plane.
type following struct {
	open *stores
	// oneShot is a process that ends after the call (a recheck through `delivery check`): what it would do beside
	// the call it does within it, or the process's exit would cut it off (novox/hq issue 726).
	oneShot bool
}

// Upgraded sends the machines running a module the version the catalogue now considers current —
// or records that they are behind, which is the default and needs no record.
//
// **Recording is not a second code path.** A machine that is not running what the mesh would send
// it is already something the mesh notices and reports; that is what `status` and `push --behind`
// are built on. So "record it" is the absence of an action, and the only thing this has to decide
// is whether to act.
func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
	// Nothing is sent: each machine follows the wants the assigner writes for the new fingerprint (ADR 0539 §5).
	fmt.Printf("%s moved to %s\n", u.Module, shortCommit(u.Commit))
	return nil
}

func shortCommit(commit string) string {
	if len(commit) > 8 {
		return commit[:8]
	}
	return commit
}

// Announceable is every build this mesh recorded, in the shape the builder announces one.
//
// **The catalogue asks for this when it starts, and the answer is the graph's foundation**
// (novox/hq 04-ISSUES/050). A durable queue keeps what arrived after it existed, so a running
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
// store the catalogue runs on, and the catalogue itself.
//
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
// the store's address, so a replay composes the address back in — the store's address as the
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
// store on the network yet, the recorded form goes as it is.
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
	builds, err := f.open.inventory.Announceable(ctx)
	if err != nil {
		return nil, err
	}
	address, err := whereTheStoreIs(ctx, f.open.inventory, "")
	if err != nil {
		return nil, err
	}
	out := make([]link.Announcement, 0, len(builds))
	for _, b := range builds {
		a := link.Announcement{
			Module: b.Module, Commit: b.Commit, Repository: b.Repository,
			Path: b.Path, Ref: b.Ref, Against: b.Against,
		}
		if len(b.Manifest) > 0 {
			a.Manifest = b.Manifest
			if address != "" {
				a.Manifest = routedManifest(b.Manifest, b.Made, address)
			}
		}
		for _, made := range routedArtifacts(b.Made, address) {
			a.Made = append(a.Made, link.MadeArtifact{
				Name: made.Name, Kind: made.Kind, Reference: made.Reference,
			})
		}
		out = append(out, a)
	}
	return out, nil
}

// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
	// A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and
	// the builder builds them (ADR 0539 §5, §17). The controller keeps the announcement, for the build of a merge that
	// moves a module between repositories to be shown the operator's Approve (ADR 0420 §7).
	return keepForgeMerge(ctx, m)
}

// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
var actingOnMerges sync.Mutex

// mergeCandidates is what one merge could move, judged against what the catalogue holds.
//
// Two kinds of module are affected by one merge, and they are affected differently.
//
// A module **built from** this repository and branch has moved: the mesh records the new commit as
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
// would make it permanently behind a repository its manifest does not come from. `already` counts
// the modules built from this repository that are already built from this very commit.
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
	read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
	for _, e := range entries {
		switch {
		case sourceIs(e.Source, m):
			if e.Source.BuiltFrom == m.Commit {
				already++
				continue
			}
			// **A merge older than the last look at the source is history, not a move.** The forge
			// announces what it finds merged, and an old merge surfacing late would otherwise move
			// the recorded head backwards and rebuild everything built from that repository, once
			// per old merge (2026-09-28).
			if isHistory(m.MergedAt, e.Source.Seen) {
				continue
			}
			from = append(from, e)
		case readsFrom(read[e.Manifest.Module], m):
			// **A merge older than the module's last look is history for it** (novox/hq ADR 0267): a
			// build or plan of it after the merge already read the repository with the merge in it. Per
			// module, since a merge that moved only the module built from the repository says nothing
			// about the ones packaging it.
			// Judged with a margin for the forge's clock running behind the store's: too late a look
			// rebuilds once more, too early one would miss the merge.
			if lookedAtCommit(read[e.Manifest.Module], m.Commit) {
				continue
			}
			if looked := lookedOf(read[e.Manifest.Module]); !looked.IsZero() &&
				isHistory(m.MergedAt, looked.Add(-historyMargin)) {
				continue
			}
			packaging = append(packaging, e)
		}
	}
	return from, packaging, already
}

// lookedAtCommit is whether a plan that built a module, or is building it, answered this merge commit.
func lookedAtCommit(read []inventory.ReadRepository, commit string) bool {
	for _, r := range read {
		if slices.Contains(r.LookedAt, commit) {
			return true
		}
	}
	return false
}

// historyMargin is how far a packaging module's last look is taken back before a merge is history for it.
const historyMargin = time.Minute

// lookedOf is when a module packaging another repository was last looked at, as readForPlanning says.
func lookedOf(read []inventory.ReadRepository) time.Time {
	var at time.Time
	for _, r := range read {
		if r.Looked.After(at) {
			at = r.Looked
		}
	}
	return at
}

// splitDeleted parts the modules a merge touched into those it changed and those whose manifest it
// removed — deleted at their source (ADR 0236).
func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) {
	removed := map[string]bool{}
	for _, p := range m.Removed {
		removed[strings.Trim(p, "/")] = true
	}
	for _, e := range touched {
		dir := strings.Trim(e.Source.Path, "/")
		manifest := moduleManifestFile
		if dir != "" {
			manifest = dir + "/" + manifest
		}
		if len(removed) > 0 && removed[manifest] {
			deleted = append(deleted, e)
			continue
		}
		kept = append(kept, e)
	}
	return kept, deleted
}

// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
// An empty recorded ref is the repository's default branch, which is what a merge into the base
// branch of the forge's default means.
func sourceIs(s inventory.Source, m link.SourceMoved) bool {
	if !sameRepository(s.Repository, m) {
		return false
	}
	return followsBase(s.Ref, m.Base)
}

// followsBase is whether a recorded ref follows the branch a merge went into. **An empty ref is the
// trunk, main, and nothing else** (novox/hq ADR 0401 clause 3, issue 889): it used to match a merge
// into any branch, so a merge into a feature branch started a delivery. A ref recorded as a branch
// keeps following that branch.
func followsBase(ref, base string) bool {
	if ref = strings.TrimSpace(followedBranch(ref)); ref == "" {
		ref = "main"
	}
	return ref == base
}

// commitRef is a ref that names a commit rather than a branch: what `build --ref <commit>` asks for.
var commitRef = regexp.MustCompile(`^[0-9a-f]{7,40}$`)

// followedBranch is the branch a recorded ref means a module follows (novox/hq 04-ISSUES/215). **A
// commit is never a branch to follow.** A build asked at a commit — to try one, or to pin it during a
// fix — recorded that commit as the module's ref; every merge after it then failed to match the
// module, its plan left it out without saying so, and every plan that rebuilt it asked for that same
// old commit again. A commit recorded so is read as the repository's default branch, which is what
// the module followed before it; a branch is followed as named.
func followedBranch(ref string) string {
	if commitRef.MatchString(strings.TrimSpace(ref)) {
		return ""
	}
	return ref
}

// sameRepository is whether a recorded repository is the one a merge names, in either spelling it
// may have been recorded in: a path on the git seat, or the URL it was cloned from.
func sameRepository(repository string, m link.SourceMoved) bool {
	want := strings.ToLower(m.Owner + "/" + m.Repo)
	repo := strings.ToLower(strings.TrimSuffix(repository, ".git"))
	return repo == want || strings.HasSuffix(repo, "/"+want) ||
		(m.CloneURL != "" && repo == strings.ToLower(strings.TrimSuffix(m.CloneURL, ".git")))
}

// readsFrom is whether a merge changed what a module's build read in another repository: the second
// repository its recipe packages source from. Its ref must be the branch that moved, or unset — the same
// rule a module's own source follows.
//
// **Only a changed file in what the build read there** (novox/hq ADR 0267 rule 2): where the module's
// newest trunk build said its build source in that repository, a merge touching none of it is no change
// to the module (issue 338: every merge to the controller's repository moved the route proxy and the
// build seat's holder). Where it said none, or the merge's files are not all said, the whole repository is
// read, as before.
func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool {
	for _, r := range read {
		if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) {
			continue
		}
		if len(r.Paths) == 0 || len(m.Paths) == 0 || m.PathsTruncated {
			return true
		}
		for _, p := range m.Paths {
			if gosource.SourceHolds(r.Paths, p) {
				return true
			}
		}
	}
	return false
}

// ownSource is the build source a module's newest trunk build said it read in its own repository; nil
// when it said none, and the module's own directory — or, built from the root, its whole repository — is
// its build source, as before (novox/hq ADR 0267).
func ownSource(read []inventory.ReadRepository) []string {
	for _, r := range read {
		if r.Own && len(r.Paths) > 0 {
			return r.Paths
		}
	}
	return nil
}

// readsFile is whether a module built from the merged repository reads one of its changed files: in its
// build source where its newest trunk build said one, else anywhere in its directory, or anywhere at all
// for a module built from the repository's root.
func readsFile(e inventory.Entry, read []inventory.ReadRepository, p string) bool {
	if own := ownSource(read); own != nil {
		return gosource.SourceHolds(own, p)
	}
	return readsDirectory(e, p)
}

// readsDirectory is whether a changed file is in a module's directory, or anywhere for a module built from its
// repository's root: the reading of a module with no recorded build source.
func readsDirectory(e inventory.Entry, p string) bool {
	return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path)
}

// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's
// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17).
func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) {
	return inv.ReadRepositories(ctx)
}

// touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the
// merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq
// ADR 0238), so planning and gating cannot disagree about what a change touches.
//
// **A changed file touches exactly the modules whose build reads it.** What a build reads is its build
// source, where the module's newest trunk build said one (novox/hq ADR 0267): a Go program's import closure,
// an archive's directory, a recipe, its manifest — so a README at the root of a repository whose module is
// built from its root, or another program's package beside it, touches nothing. Where none was said, it is
// the module's own directory — the builder clones the repository and builds within that directory alone: the manifest,
// the recipes, the bundles' sources, the Docker context — or the whole repository for a module built from
// its root. A second repository a recipe packages (an artifact's `context`) is read too; that is the build
// record's `read`, answered by readsFrom in mergeCandidates. So a changed file inside a module's directory
// is that module's; a file in no module's directory — a script at the root, a README, CI configuration —
// is read by no build and touches nothing: it is answered as `unread`.
//
// **It used to be read as shared code**, rebuilding everything built from the repository, on the theory
// that a root file might be a build input (04-ISSUES/131). No build reads one: the theory cost a rebuild
// of 103 modules for a merge-check.sh added at the catalogue's root (issue 280), as it had for a module
// held by no machine (278) and a new module's directory (252), each patched as an exception to a rule that
// was wrong. Rebuilding too much is not the safe direction when every rebuild is a rollout.
//
// `added` is the directories the change holds a module in that no module of this repository is known
// from — said by the announcer at the head (issue 278), or a module.json among the changed files — `.`
// for the root: a new module, which a check judges before it merges and the merge builds and registers,
// sending it nowhere (askNewModules, novox/hq issue 300).
//
// Nothing said about the files, or not all of them said, is still everything: what is not known cannot
// be narrowed.
func touchedBy(candidates, known []inventory.Entry, m link.SourceMoved,
	read map[string][]inventory.ReadRepository) (touched []inventory.Entry, added, unread []string) {
	knownDirs := map[string]bool{}
	for _, e := range known {
		if !e.Provided && sameRepository(e.Source.Repository, m) {
			knownDirs[strings.Trim(e.Source.Path, "/")] = true
		}
	}
	newDir := map[string]bool{}
	for _, d := range saidModuleDirs(m) {
		if !knownDirs[d] {
			newDir[d] = true
		}
	}
	for _, p := range m.Paths {
		p = strings.Trim(p, "/")
		if p != moduleManifestFile && !strings.HasSuffix(p, "/"+moduleManifestFile) {
			continue
		}
		d := strings.TrimSuffix(strings.TrimSuffix(p, moduleManifestFile), "/")
		if !knownDirs[d] {
			if d == "" {
				d = "."
			}
			newDir[d] = true
		}
	}
	for d := range newDir {
		added = append(added, d)
	}
	sort.Strings(added)

	if len(m.Paths) == 0 || m.PathsTruncated {
		return candidates, added, nil
	}
	for _, e := range candidates {
		for _, p := range m.Paths {
			if readsFile(e, read[e.Manifest.Module], p) {
				touched = append(touched, e)
				break
			}
		}
	}
	for _, p := range m.Paths {
		isRead := newDir["."]
		for _, e := range candidates {
			isRead = isRead || readsFile(e, read[e.Manifest.Module], p)
		}
		for d := range newDir {
			isRead = isRead || inside(p, d)
		}
		// A file a module packages from this repository is read too, by that module's build.
		for _, e := range known {
			isRead = isRead || readsFrom(read[e.Manifest.Module], link.SourceMoved{Owner: m.Owner, Repo: m.Repo,
				Base: m.Base, CloneURL: m.CloneURL, Paths: []string{p}})
		}
		if !isRead {
			unread = append(unread, p)
		}
	}
	return touched, added, unread
}

// mergeReach is what a merge of a repository's branch reaches, as the planner reckons it: the planner's
// one answer, given to the release planner's what-if, the merge gate and a pull request's check (novox/hq
// ADR 0238). The merge handler acts on the same pieces — mergeCandidates, touchedBy, splitDeleted,
// planOfMerge — as it goes.
type mergeReach struct {
	// Touched are the modules built from the repository and branch whose build reads a changed file, and
	// Deleted those of them whose manifest the merge removes.
	Touched, Deleted []inventory.Entry
	// Packaging are the modules whose build packages source from the repository.
	Packaging []inventory.Entry
	// Already counts the modules built from the repository that are built from this very commit.
	Already int
	// Added are the directories the change holds a new module in; Unread the changed files no build reads.
	Added, Unread []string
	// Following are the modules that stand on the moved ones along the catalogue's dependencies: what a merge would
	// build after them.
	Following []string
}

// Moved are the modules the merge moves itself, by name: touched, deleted and packaging.
func (r mergeReach) Moved() []string {
	var out []string
	for _, group := range [][]inventory.Entry{r.Touched, r.Deleted, r.Packaging} {
		for _, e := range group {
			out = append(out, e.Manifest.Module)
		}
	}
	sort.Strings(out)
	return slices.Compact(out)
}

// Dependents are the modules the plan builds after the moved ones because they stand on them.
func (r mergeReach) Dependents() []string {
	return r.Following
}

// saidModuleDirs is the directories the announcer says hold a module at the merge commit (novox/hq
// issue 278): a changed file in one of them is that module's business and nobody else's — the
// catalogue's reference module, held by no machine, whose code a merge touched beside its manifest,
// read as shared and rebuilt 103 modules. Nothing when the announcer did not look, and never the
// repository's root: a module built from the root is handled as one built from it, and a root that
// counted would make every file a module's.
func saidModuleDirs(m link.SourceMoved) []string {
	if !m.ModuleDirsSaid {
		return nil
	}
	var out []string
	for _, d := range m.ModuleDirs {
		if d = strings.Trim(strings.TrimSpace(d), "/"); d != "" && d != "." {
			out = append(out, d)
		}
	}
	return out
}

// inside is whether a changed file is in a directory: that directory itself, or under it.
func inside(path, dir string) bool {
	dir = strings.Trim(dir, "/")
	path = strings.TrimPrefix(path, "/")
	return path == dir || strings.HasPrefix(path, dir+"/")
}

// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
// with no time on it is taken as news: refusing it would silence a forge that says less.
func isHistory(mergedAt string, seen time.Time) bool {
	if mergedAt == "" || seen.IsZero() {
		return false
	}
	at, err := time.Parse(time.RFC3339, mergedAt)
	if err != nil {
		return false
	}
	return at.Before(seen)
}

// moduleManifestFile is the file that says what a module is, in its directory (the build seat's
// ManifestName).
const moduleManifestFile = "module.json"

// reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's
// reach, as the check reckons it.
func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
	edges []inventory.Edge) mergeReach {
	from, packaging, already := mergeCandidates(m, entries, read)
	touched, added, unread := touchedBy(from, entries, m, read)
	kept, deleted := splitDeleted(touched, m)
	r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread}
	moved := r.Moved()
	if len(moved) > 0 {
		in := map[string]bool{}
		for _, name := range moved {
			in[name] = true
		}
		for _, name := range reachableFrom(moved, edges) {
			if !in[name] {
				r.Following = append(r.Following, name)
			}
		}
	}
	return r
}

// reachableFrom is the moved modules plus everything that depends on them, through every layer:
// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine
// is not changed by a new build machine, so a built-by edge orders and gates a plan and never
// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change.
func reachableFrom(moved []string, edges []inventory.Edge) []string {
	in := map[string]bool{}
	for _, m := range moved {
		in[m] = true
	}
	for grew := true; grew; {
		grew = false
		for _, e := range edges {
			// Built-by and worker-of order a plan; neither widens it. A new build machine changes
			// nothing it builds, and a new controller changes nothing about the holder it orders. A
			// packages edge, read from a record made before novox/hq ADR 0267, widens nothing either:
			// a shared file moves each module whose build source holds it, directly.
			if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages ||
				e.Kind == edgeChangeOrder {
				continue
			}
			if in[e.To] && !in[e.From] {
				in[e.From] = true
				grew = true
			}
		}
	}
	out := make([]string, 0, len(in))
	for m := range in {
		out = append(out, m)
	}
	sort.Strings(out)
	return out
}

// edgeChangeOrder is the edge kind a change order draws.
const edgeChangeOrder = "change-order"

// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR
// 0235): every module's policy and where it comes from; one module's; or a person's choice for one.
func upgradeCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
	together := set.Bool("together", false,
		"send every machine running it at once, instead of one machine first")
	why := set.String("why", "", "why this choice — required for record; kept and said with the policy")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	if len(positionals) == 0 {
		all, err := inv.Upgrades(ctx)
		if err != nil {
			return err
		}
		names := make([]string, 0, len(all))
		for n := range all {
			names = append(names, n)
		}
		sort.Strings(names)
		for _, n := range names {
			u := all[n]
			line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From)
			if u.Why != "" {
				line += ": " + u.Why
			}
			fmt.Println(line)
		}
		return nil
	}
	module := positionals[0]
	if len(positionals) == 1 {
		decision, err := inv.UpgradeOf(ctx, module)
		if err != nil {
			return err
		}
		fmt.Println(sayUpgrade(module, decision))
		return nil
	}

	decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()}
	switch positionals[1] {
	case "roll-out", "roll":
		decision.RollOut, decision.Together = true, *together
	case "record":
		if *together {
			// Refused rather than ignored: --together only means anything for a roll-out, and
			// accepting it here would store a preference that never applies and looks like it does.
			return errors.New("`--together` says how to roll out, so it cannot be given with " +
				"`record`, which is the choice not to")
		}
		if decision.Why == "" {
			return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why <text> "+
				"(novox/hq ADR 0236). Nothing was changed", module)
		}
	case "default":
		if err := inv.ClearUpgradeOf(ctx, module); err != nil {
			return err
		}
		decision, err := inv.UpgradeOf(ctx, module)
		if err != nil {
			return err
		}
		fmt.Println(sayUpgrade(module, decision))
		// Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14).
		rejudgeModule(ctx, open, module)
		return nil
	default:
		return fmt.Errorf("upgrade <module> roll-out|record|default — not %q", positionals[1])
	}
	if err := inv.SetUpgradeOf(ctx, module, decision); err != nil {
		return err
	}
	decision, err = inv.UpgradeOf(ctx, module)
	if err != nil {
		return err
	}
	fmt.Println(sayUpgrade(module, decision))
	// Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14).
	rejudgeModule(ctx, open, module)
	return nil
}

func sayUpgrade(module string, u inventory.Upgrade) string {
	from := " (" + u.From
	if u.By != "" {
		from += ", " + u.By
	}
	if u.Why != "" {
		from += ": " + u.Why
	}
	from += ")"
	if !u.RollOut {
		return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+
			"reported as behind until a person pushes them%s", module, from)
	}
	if u.Together {
		return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+
			"together%s", module, from)
	}
	return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+
		"the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from)
}