package main import ( "context" "flag" "fmt" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/overlay" ) // is anything broken, is anything not answering, is anything out of date. // // Split out of main.go, which had reached 2,769 lines because appending was always the // cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: // nothing in it was wrong, and no one edit was the one that should have been a new file. // short is a commit as a person refers to it. func short(commit string) string { if len(commit) > 8 { return commit[:8] } return commit } // statusCommand answers "did my change go out?". // // novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a // comparison: it is answerable today by opening a pipeline, and something has to replace that or // this is worse to live with whatever its other properties. // // The answer is not "a job succeeded". It is which modules the mesh has not built from what their // source now has, and which machines are running the old one. func statusCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("status", flag.ContinueOnError) asJSON := set.Bool("json", false, "the same answers, for something other than a person") if _, err := parseAround(set, args); err != nil { return err } open, err := openStores(ctx) if err != nil { return err } defer open.Close() return statusFor(ctx, open, *asJSON) } // statusFor asks and answers, against stores somebody else opened. // // Split from the command so what it prints can be read by a test. The sentence it prints when nothing // is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact // words the thing worth holding still. func statusFor(ctx context.Context, open *stores, asJSON bool) error { asked, err := theThreeQuestions(ctx, open) if err != nil { return err } if asJSON { body, err := statusAsJSON(asked) if err != nil { return err } fmt.Println(string(body)) return nil } return printStatus(asked) } // printStatus is the words, separated from the questions. // // **Its exact sentences have been acted on and been misleading twice** — a held module reading as a // machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being // true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a // test can read them without a store, a bus or a machine. func printStatus(asked answers) error { wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet behind, sources := asked.behind, asked.sources if len(asked.refused) > 0 { // First, above everything else. A machine that cannot be worked out is not running an old // declaration — it has no declaration, and nothing below this line is about it. var names []string for name := range asked.refused { names = append(names, name) } sort.Strings(names) fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n", len(names)) for _, name := range names { fmt.Printf(" %s\n", name) for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") { fmt.Printf(" %s\n", strings.TrimSpace(line)) } } fmt.Println() } if asked.network != "" { fmt.Printf("the private network could not be computed:\n %s\n\n", strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n ")) } if len(wrong) > 0 { fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong)) for _, d := range wrong { fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04")) if d.Refused != "" { // The host's own words. It says exactly what it could not accept, and nothing // written here would say it better. fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused)) } for _, f := range d.Failed { fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error)) } if d.Stuck() { // Said apart from the failure itself. The host's words say what is wrong; this // says it is not new — the machine has applied, failed the same way and reported // so this many times, and will keep doing exactly that until something changes // (novox/hq 04-ISSUES/065). fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n", "", d.Times, d.Since.Local().Format("2006-01-02 15:04")) } } fmt.Println() } if len(quiet) > 0 { var said []string for _, n := range quiet { said = append(said, n.Name+" ("+heardFrom(n)+")") } fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n", len(quiet), strings.Join(said, "\n ")) } if len(behind) > 0 { var names []string for m := range behind { names = append(names, m) } sort.Strings(names) fmt.Printf("%d module(s) behind their source:\n\n", len(behind)) for _, m := range names { from := sources[m] fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head)) if on := behind[m]; len(on) > 0 { // The part somebody actually wants. A module being out of date is a fact about // the catalogue; machines running the old one is the thing with consequences. fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", ")) } else { fmt.Printf(" %-18s assigned to nothing\n", "") } } // The remedy, beside the problem. A status that says what is wrong and not what to do // about it makes somebody go and find the command, and the command is the whole point of // having noticed. fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n") fmt.Println() } if len(asked.waiting) > 0 { // The other half of "is anything out of date": a module behind its source says the // catalogue is old, and this says a machine is — and only this one has somebody's change // waiting inside it. var told, never []string for _, m := range asked.waiting { if m.Never { never = append(never, m.Node) continue } told = append(told, m.Node) } if len(told) > 0 { fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n", len(told), strings.Join(told, ", ")) } if len(never) > 0 { // Never told is not out of date. The remedy is the same push and the situation is // not the same at all: nobody has ever asked this machine to be anything. fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", len(never), strings.Join(never, ", ")) } fmt.Printf("\n `push --behind` sends them\n\n") } if split := hostSplit(nodes); len(split) > 1 { // **Before a declaration gains a field, every machine has to understand it** (novox/hq // 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses // it whole, so every new field is a flag day: hosts first, then the controller. The mesh had // no record of which host any machine ran, so that order was kept by somebody remembering it. // // **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and // commits have no order — the first version of this said "N machines run an older host" and // named the three that were newer, because it compared two hashes as strings. What the mesh // can say truthfully is that the machines do not all run the same host, and which machines // hold which. Ordering needs a version that is ordered, and that is the host's to report. versions := make([]string, 0, len(split)) for v := range split { versions = append(versions, v) } sort.Strings(versions) fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes)) for _, v := range versions { sort.Strings(split[v]) fmt.Printf(" %-12s %s\n", v, strings.Join(split[v], ", ")) } fmt.Printf("\n a host refuses a declaration carrying a field it does not know, whole — so the\n" + " mesh may send only what every one of these understands. Which of them is newer is\n" + " not readable from a commit; that needs a version the host reports as ordered\n\n") } if len(asked.untaken) > 0 { // **Before the adopted line, and it breaks "all well".** An adopted machine is a state // somebody chose and can leave alone; a module assigned to one and never taken is work // outstanding that reads exactly like work finished. That reading is what stopped a // predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125). machines := make([]string, 0, len(asked.untaken)) for name := range asked.untaken { machines = append(machines, name) } sort.Strings(machines) total := 0 for _, held := range asked.untaken { for _, n := range held { total += n } } fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+ "taken — so it is running none of what it declares:\n", total) for _, name := range machines { modules := make([]string, 0, len(asked.untaken[name])) for m := range asked.untaken[name] { modules = append(modules, m) } sort.Strings(modules) parts := make([]string, 0, len(modules)) for _, m := range modules { parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m])) } fmt.Printf(" %-12s %s\n", name, strings.Join(parts, ", ")) } fmt.Printf("\n `take ` compares what runs against what it declares, and runs it\n\n") } if adopted := adoptedNodes(nodes); len(adopted) > 0 { // Said, because nothing forces the flip: a node left adopted is visible here rather than // read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well". fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", ")) fmt.Printf("\n `converge ` previews the flip\n\n") } if asked.well() { // Said plainly. "Nothing to report" and "nothing was checked" must never look the same, // and getting here means every question was asked and answered. fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+ "the mesh would send them, and every module current with its source\n", len(nodes)) // **And what that sentence does not cover**, because for eleven hours it was true of a mesh // in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is // about the relationship between the mesh and a machine — applied what it was sent, matches // what would be sent, built from what the source has. None of them asks whether a module can // reach what it requires, and the mesh composes every one of those grants itself. // // Said here rather than left to be inferred. A reader who acts on the line above is acting on // "the machines are as the mesh described them", and the distance between that and "it works" // is where the eleven hours went. fmt.Printf("\n That is the mesh and the machines agreeing. Nothing here dials a provision:\n" + " no grant the mesh composed has been tested, so a module unable to reach what it\n" + " requires would not appear above (04-ISSUES/145)\n") } return nil } // firstLine is as much of a failure as belongs in a list. func firstLine(s string) string { if cut := strings.IndexByte(s, '\n'); cut >= 0 { return strings.TrimSpace(s[:cut]) } return strings.TrimSpace(s) } // builds keeps what a builder said, for the serving control plane. // // A type of its own rather than a method on the enrolment, because they are unrelated things // arriving on one queue and an implementation of one should not have to say anything about the // other. type builds struct{ inv *inventory.Inventory } // theThreeQuestions reads what anything answering "is the mesh alright" needs. // // **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There // are three now — a person's status, its JSON, and a page — and three implementations of "which // machine is not doing what it was told" would be three chances to disagree about it. // // The order is the design and not a convenience: is anything broken, is anything not answering, is // anything out of date. The first has consequences now, the second may, the third is a plan for // later — and anything that led with the third would bury the first. func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { inv := open.inventory var out answers var err error out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx) if err != nil { return answers{}, err } out.nodes, err = inv.Nodes(ctx) if err != nil { return answers{}, err } for _, n := range out.nodes { // Never heard from, or not lately. Different from failing: a machine that says nothing // may be new, switched off, or unreachable, and none of those is a machine that tried // and could not. if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour { out.quiet = append(out.quiet, n) } } out.behind, err = inv.Behind(ctx) if err != nil { return answers{}, err } // And why any machine cannot be worked out at all, which is neither of the first two questions // and is asked before them both in practice: a machine nothing can be computed for is not // broken, not quiet and not behind, and every other answer here would call it well. // // Read through whoResolves, which is what the private network is built from, so this and the // network agree about who could not be resolved rather than deciding it twice. _, out.refused, err = whoResolves(ctx, open, overlay.Addressing) if err != nil { return answers{}, err } // And what each machine is holding rather than running, by the module that would run it. Read // from what the machine itself last reported, not from what take-time computed: the machine is // the only thing that knows what it found (novox/hq 04-ISSUES/125). out.untaken, err = untakenModules(ctx, inv, out.nodes) if err != nil { return answers{}, err } // And which machines are not running what the mesh would send them. The same question as a // module being behind its source, one level down: that one says the catalogue is out of date, // this one says a machine is — and only the second has anybody's change waiting in it. // // **One machine that cannot be resolved must not take the answer away from every other** // (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub // is the blocked machine has no hub — which used to come back here as a refusal, so `status` // said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The // reason is kept and reported as data; every question that does not depend on it is still // answered. would, err := wouldSend(ctx, open, out.nodes) if err != nil { out.network = err.Error() would = map[string]string{} } out.waiting, err = inv.Waiting(ctx, would) if err != nil { return answers{}, err } out.reported, err = inv.LastReports(ctx) if err != nil { return answers{}, err } out.sources = map[string]inventory.Source{} for module := range out.behind { from, err := inv.SourceOf(ctx, module) if err != nil { return answers{}, err } out.sources[module] = from } return out, nil } // untakenModules is, per machine, each module whose resources that machine is holding as found, and // how many. // // **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found // and reports it; the mesh's take-time listing is a different thing and was the one this command used // to have, which is why a module assigned after the listing showed nothing at all // (novox/hq 04-ISSUES/125). // // A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and // the caller prints nothing. func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) ( map[string]map[string]int, error) { out := map[string]map[string]int{} for _, n := range nodes { said, err := inv.AdoptionOf(ctx, n.Name) if err != nil { // A machine whose record cannot be read is not a machine holding nothing. Said, because // answering "nothing held" from a failed read is the shape this whole issue is about. return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err) } for _, h := range said.Held { if h.Module == "" { continue // a hold the mesh cannot attribute to a module has nothing to take } if out[n.Name] == nil { out[n.Name] = map[string]int{} } out[n.Name][h.Module]++ } } return out, nil } // well is whether every question this command asks came back with nothing to say. // // Named, and in one place, because it is the sentence an operator acts on and it has been wrong // twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken // and is not doing what it was told either. // // **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave // alone. A module assigned to a machine and never taken is a half-finished action with nothing left // to finish it — it runs none of what it declares, and "all doing what they were told" was true and // read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125). func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 } // hostSplit is which machines report which host version, for every version more than one machine // could disagree about. // // **It does not say which is newer, because it cannot.** A host reports its version as a commit, and // commits have no order. The first version of this returned "the machines behind the newest" by // comparing versions as strings, and on the live mesh it named the three machines running the NEWER // host as the ones behind — an arbitrary lexicographic result presented as a fact // (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one // that says less, which is the whole subject of 04-ISSUES/145. // // So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no // ordering. Ordering wants an ordered version, and that is the host's to report rather than this // function's to infer. // // Machines that have not reported a version are left out entirely: they are not a version, and // counting them as one would invent a disagreement. `node show` says per machine that it has not said. func hostSplit(nodes []inventory.Node) map[string][]string { out := map[string][]string{} for _, n := range nodes { if n.HostVersion == "" { continue } out[n.HostVersion] = append(out[n.HostVersion], n.Name) } if len(out) < 2 { return nil // one version, or none reported: nothing to disagree about } return out }