mesh-controller/cmd/mesh-controller/status.go

package main

import (
	"context"
	"flag"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/overlay"
)

// is anything broken, is anything not answering, is anything out of date.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.

// short is a commit as a person refers to it.
func short(commit string) string {
	if len(commit) > 8 {
		return commit[:8]
	}
	return commit
}

// statusCommand answers "did my change go out?".
//
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
// this is worse to live with whatever its other properties.
//
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
// source now has, and which machines are running the old one.
func statusCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("status", flag.ContinueOnError)
	asJSON := set.Bool("json", false, "the same answers, for something other than a person")
	if _, err := parseAround(set, args); err != nil {
		return err
	}

	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	return statusFor(ctx, open, *asJSON)
}

// statusFor asks and answers, against stores somebody else opened.
//
// Split from the command so what it prints can be read by a test. The sentence it prints when nothing
// is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact
// words the thing worth holding still.
func statusFor(ctx context.Context, open *stores, asJSON bool) error {
	asked, err := theThreeQuestions(ctx, open)
	if err != nil {
		return err
	}

	if asJSON {
		body, err := statusAsJSON(asked)
		if err != nil {
			return err
		}
		fmt.Println(string(body))
		return nil
	}
	return printStatus(asked)
}

// printStatus is the words, separated from the questions.
//
// **Its exact sentences have been acted on and been misleading twice** — a held module reading as a
// machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being
// true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a
// test can read them without a store, a bus or a machine.
func printStatus(asked answers) error {
	wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
	behind, sources := asked.behind, asked.sources

	if len(asked.refused) > 0 {
		// First, above everything else. A machine that cannot be worked out is not running an old
		// declaration — it has no declaration, and nothing below this line is about it.
		var names []string
		for name := range asked.refused {
			names = append(names, name)
		}
		sort.Strings(names)
		fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
			len(names))
		for _, name := range names {
			fmt.Printf("  %s\n", name)
			for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
				fmt.Printf("    %s\n", strings.TrimSpace(line))
			}
		}
		fmt.Println()
	}

	if asked.network != "" {
		fmt.Printf("the private network could not be computed:\n  %s\n\n",
			strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n  "))
	}

	if len(wrong) > 0 {
		fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
		for _, d := range wrong {
			fmt.Printf("  %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04"))
			if d.Refused != "" {
				// The host's own words. It says exactly what it could not accept, and nothing
				// written here would say it better.
				fmt.Printf("  %-18s %s\n", "", firstLine(d.Refused))
			}
			for _, f := range d.Failed {
				fmt.Printf("  %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
			}
			if d.Stuck() {
				// Said apart from the failure itself. The host's words say what is wrong; this
				// says it is not new — the machine has applied, failed the same way and reported
				// so this many times, and will keep doing exactly that until something changes
				// (novox/hq 04-ISSUES/065).
				fmt.Printf("  %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
					"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
			}
		}
		fmt.Println()
	}

	if len(quiet) > 0 {
		var said []string
		for _, n := range quiet {
			said = append(said, n.Name+" ("+heardFrom(n)+")")
		}
		fmt.Printf("%d machine(s) not heard from lately:\n  %s\n\n",
			len(quiet), strings.Join(said, "\n  "))
	}

	if len(behind) > 0 {
		var names []string
		for m := range behind {
			names = append(names, m)
		}
		sort.Strings(names)

		fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
		for _, m := range names {
			from := sources[m]
			fmt.Printf("  %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
			if on := behind[m]; len(on) > 0 {
				// The part somebody actually wants. A module being out of date is a fact about
				// the catalogue; machines running the old one is the thing with consequences.
				fmt.Printf("  %-18s running on %s\n", "", strings.Join(on, ", "))
			} else {
				fmt.Printf("  %-18s assigned to nothing\n", "")
			}
		}
		// The remedy, beside the problem. A status that says what is wrong and not what to do
		// about it makes somebody go and find the command, and the command is the whole point of
		// having noticed.
		fmt.Printf("\n  `build --behind` builds them; `push --behind` sends them on\n")
		fmt.Println()
	}

	if len(asked.waiting) > 0 {
		// The other half of "is anything out of date": a module behind its source says the
		// catalogue is old, and this says a machine is — and only this one has somebody's change
		// waiting inside it.
		var told, never []string
		for _, m := range asked.waiting {
			if m.Never {
				never = append(never, m.Node)
				continue
			}
			told = append(told, m.Node)
		}
		if len(told) > 0 {
			fmt.Printf("%d machine(s) are not running what the mesh would send them:\n  %s\n",
				len(told), strings.Join(told, ", "))
		}
		if len(never) > 0 {
			// Never told is not out of date. The remedy is the same push and the situation is
			// not the same at all: nobody has ever asked this machine to be anything.
			fmt.Printf("%d machine(s) have never been sent anything:\n  %s\n",
				len(never), strings.Join(never, ", "))
		}
		fmt.Printf("\n  `push --behind` sends them\n\n")
	}

	if split := hostSplit(nodes); len(split) > 1 {
		// **Before a declaration gains a field, every machine has to understand it** (novox/hq
		// 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses
		// it whole, so every new field is a flag day: hosts first, then the controller. The mesh had
		// no record of which host any machine ran, so that order was kept by somebody remembering it.
		//
		// **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and
		// commits have no order — the first version of this said "N machines run an older host" and
		// named the three that were newer, because it compared two hashes as strings. What the mesh
		// can say truthfully is that the machines do not all run the same host, and which machines
		// hold which. Ordering needs a version that is ordered, and that is the host's to report.
		versions := make([]string, 0, len(split))
		for v := range split {
			versions = append(versions, v)
		}
		sort.Strings(versions)
		fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes))
		for _, v := range versions {
			sort.Strings(split[v])
			fmt.Printf("  %-12s %s\n", v, strings.Join(split[v], ", "))
		}
		fmt.Printf("\n  a host refuses a declaration carrying a field it does not know, whole — so the\n" +
			"  mesh may send only what every one of these understands. Which of them is newer is\n" +
			"  not readable from a commit; that needs a version the host reports as ordered\n\n")
	}

	if len(asked.untaken) > 0 {
		// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
		// somebody chose and can leave alone; a module assigned to one and never taken is work
		// outstanding that reads exactly like work finished. That reading is what stopped a
		// predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125).
		machines := make([]string, 0, len(asked.untaken))
		for name := range asked.untaken {
			machines = append(machines, name)
		}
		sort.Strings(machines)
		total := 0
		for _, held := range asked.untaken {
			for _, n := range held {
				total += n
			}
		}
		fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+
			"taken — so it is running none of what it declares:\n", total)
		for _, name := range machines {
			modules := make([]string, 0, len(asked.untaken[name]))
			for m := range asked.untaken[name] {
				modules = append(modules, m)
			}
			sort.Strings(modules)
			parts := make([]string, 0, len(modules))
			for _, m := range modules {
				parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m]))
			}
			fmt.Printf("  %-12s %s\n", name, strings.Join(parts, ", "))
		}
		fmt.Printf("\n  `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
	}

	if adopted := adoptedNodes(nodes); len(adopted) > 0 {
		// Said, because nothing forces the flip: a node left adopted is visible here rather than
		// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
		fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
		fmt.Printf("\n  `converge <node>` previews the flip\n\n")
	}

	if asked.well() {
		// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
		// and getting here means every question was asked and answered.
		fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
			"the mesh would send them, and every module current with its source\n", len(nodes))
		// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
		// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
		// about the relationship between the mesh and a machine — applied what it was sent, matches
		// what would be sent, built from what the source has. None of them asks whether a module can
		// reach what it requires, and the mesh composes every one of those grants itself.
		//
		// Said here rather than left to be inferred. A reader who acts on the line above is acting on
		// "the machines are as the mesh described them", and the distance between that and "it works"
		// is where the eleven hours went.
		fmt.Printf("\n  That is the mesh and the machines agreeing. Nothing here dials a provision:\n" +
			"  no grant the mesh composed has been tested, so a module unable to reach what it\n" +
			"  requires would not appear above (04-ISSUES/145)\n")
	}
	return nil
}

// firstLine is as much of a failure as belongs in a list.
func firstLine(s string) string {
	if cut := strings.IndexByte(s, '\n'); cut >= 0 {
		return strings.TrimSpace(s[:cut])
	}
	return strings.TrimSpace(s)
}

// builds keeps what a builder said, for the serving control plane.
//
// A type of its own rather than a method on the enrolment, because they are unrelated things
// arriving on one queue and an implementation of one should not have to say anything about the
// other.
type builds struct{ inv *inventory.Inventory }

// theThreeQuestions reads what anything answering "is the mesh alright" needs.
//
// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There
// are three now — a person's status, its JSON, and a page — and three implementations of "which
// machine is not doing what it was told" would be three chances to disagree about it.
//
// The order is the design and not a convenience: is anything broken, is anything not answering, is
// anything out of date. The first has consequences now, the second may, the third is a plan for
// later — and anything that led with the third would bury the first.
func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
	inv := open.inventory
	var out answers
	var err error

	out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx)
	if err != nil {
		return answers{}, err
	}
	out.nodes, err = inv.Nodes(ctx)
	if err != nil {
		return answers{}, err
	}
	for _, n := range out.nodes {
		// Never heard from, or not lately. Different from failing: a machine that says nothing
		// may be new, switched off, or unreachable, and none of those is a machine that tried
		// and could not.
		if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
			out.quiet = append(out.quiet, n)
		}
	}
	out.behind, err = inv.Behind(ctx)
	if err != nil {
		return answers{}, err
	}
	// And why any machine cannot be worked out at all, which is neither of the first two questions
	// and is asked before them both in practice: a machine nothing can be computed for is not
	// broken, not quiet and not behind, and every other answer here would call it well.
	//
	// Read through whoResolves, which is what the private network is built from, so this and the
	// network agree about who could not be resolved rather than deciding it twice.
	_, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
	if err != nil {
		return answers{}, err
	}
	// And what each machine is holding rather than running, by the module that would run it. Read
	// from what the machine itself last reported, not from what take-time computed: the machine is
	// the only thing that knows what it found (novox/hq 04-ISSUES/125).
	out.untaken, err = untakenModules(ctx, inv, out.nodes)
	if err != nil {
		return answers{}, err
	}

	// And which machines are not running what the mesh would send them. The same question as a
	// module being behind its source, one level down: that one says the catalogue is out of date,
	// this one says a machine is — and only the second has anybody's change waiting in it.
	//
	// **One machine that cannot be resolved must not take the answer away from every other**
	// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
	// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
	// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
	// reason is kept and reported as data; every question that does not depend on it is still
	// answered.
	would, err := wouldSend(ctx, open, out.nodes)
	if err != nil {
		out.network = err.Error()
		would = map[string]string{}
	}
	out.waiting, err = inv.Waiting(ctx, would)
	if err != nil {
		return answers{}, err
	}
	out.reported, err = inv.LastReports(ctx)
	if err != nil {
		return answers{}, err
	}
	out.sources = map[string]inventory.Source{}
	for module := range out.behind {
		from, err := inv.SourceOf(ctx, module)
		if err != nil {
			return answers{}, err
		}
		out.sources[module] = from
	}
	return out, nil
}

// untakenModules is, per machine, each module whose resources that machine is holding as found, and
// how many.
//
// **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found
// and reports it; the mesh's take-time listing is a different thing and was the one this command used
// to have, which is why a module assigned after the listing showed nothing at all
// (novox/hq 04-ISSUES/125).
//
// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
// the caller prints nothing.
func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
	map[string]map[string]int, error) {

	out := map[string]map[string]int{}
	for _, n := range nodes {
		said, err := inv.AdoptionOf(ctx, n.Name)
		if err != nil {
			// A machine whose record cannot be read is not a machine holding nothing. Said, because
			// answering "nothing held" from a failed read is the shape this whole issue is about.
			return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err)
		}
		for _, h := range said.Held {
			if h.Module == "" {
				continue // a hold the mesh cannot attribute to a module has nothing to take
			}
			if out[n.Name] == nil {
				out[n.Name] = map[string]int{}
			}
			out[n.Name][h.Module]++
		}
	}
	return out, nil
}

// well is whether every question this command asks came back with nothing to say.
//
// Named, and in one place, because it is the sentence an operator acts on and it has been wrong
// twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken
// and is not doing what it was told either.
//
// **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave
// alone. A module assigned to a machine and never taken is a half-finished action with nothing left
// to finish it — it runs none of what it declares, and "all doing what they were told" was true and
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
func (a answers) well() bool {
	return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
		len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0
}

// hostSplit is which machines report which host version, for every version more than one machine
// could disagree about.
//
// **It does not say which is newer, because it cannot.** A host reports its version as a commit, and
// commits have no order. The first version of this returned "the machines behind the newest" by
// comparing versions as strings, and on the live mesh it named the three machines running the NEWER
// host as the ones behind — an arbitrary lexicographic result presented as a fact
// (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one
// that says less, which is the whole subject of 04-ISSUES/145.
//
// So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no
// ordering. Ordering wants an ordered version, and that is the host's to report rather than this
// function's to infer.
//
// Machines that have not reported a version are left out entirely: they are not a version, and
// counting them as one would invent a disagreement. `node show` says per machine that it has not said.
func hostSplit(nodes []inventory.Node) map[string][]string {
	out := map[string][]string{}
	for _, n := range nodes {
		if n.HostVersion == "" {
			continue
		}
		out[n.HostVersion] = append(out[n.HostVersion], n.Name)
	}
	if len(out) < 2 {
		return nil // one version, or none reported: nothing to disagree about
	}
	return out
}